From 161662d95c42eb25cb3b16cec1afaa5ab6bdb187 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 14 Mar 2024 13:12:51 -0400
Subject: [PATCH 001/132] initial relational db sink

---
 stix2/datastore/relational_db/__init__.py     |  18 +
 .../postgres_database_connection.py           |  12 +
 .../datastore/relational_db/relational_db.py  |  87 ++++
 .../relational_db/relational_db_testing.py    | 112 +++++
 .../relational_db/sql_bindings_creation.py    | 403 ++++++++++++++++++
 5 files changed, 632 insertions(+)
 create mode 100644 stix2/datastore/relational_db/__init__.py
 create mode 100644 stix2/datastore/relational_db/postgres_database_connection.py
 create mode 100644 stix2/datastore/relational_db/relational_db.py
 create mode 100644 stix2/datastore/relational_db/relational_db_testing.py
 create mode 100644 stix2/datastore/relational_db/sql_bindings_creation.py

diff --git a/stix2/datastore/relational_db/__init__.py b/stix2/datastore/relational_db/__init__.py
new file mode 100644
index 00000000..1dcd772e
--- /dev/null
+++ b/stix2/datastore/relational_db/__init__.py
@@ -0,0 +1,18 @@
+from abc import abstractmethod
+
+
+class DatabaseConnection():
+
+    def __init__(self):
+        pass
+
+    @abstractmethod
+    def execute(self, sql_statement, bindings):
+        """
+
+        Args:
+            bindings:
+
+        Returns:
+
+        """
diff --git a/stix2/datastore/relational_db/postgres_database_connection.py b/stix2/datastore/relational_db/postgres_database_connection.py
new file mode 100644
index 00000000..3c92a058
--- /dev/null
+++ b/stix2/datastore/relational_db/postgres_database_connection.py
@@ -0,0 +1,12 @@
+import postgres
+
+from stix2.datastore.relational_db import DatabaseConnection
+
+
+class PostgresDatabaseConnection(DatabaseConnection):
+
+    def __init__(self, host, dbname, user):
+        self.db = postgres.Postgres(url=f"host={host} dbname={dbname} user={user}")
+
+    def execute(self, sql_statement, bindings):
+        self.db.run(sql_statement, parameters=bindings)
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
new file mode 100644
index 00000000..28f0191d
--- /dev/null
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -0,0 +1,87 @@
+from stix2 import v20, v21
+from stix2.base import _STIXBase, _Observable
+from stix2.datastore import DataSink
+from stix2.parsing import parse
+from stix2.datastore.relational_db.sql_bindings_creation import generate_insert_for_object
+
+
+def _add(store, stix_data, allow_custom=True, version=None):
+    """Add STIX objects to MemoryStore/Sink.
+
+    Adds STIX objects to an in-memory dictionary for fast lookup.
+    Recursive function, breaks down STIX Bundles and lists.
+
+    Args:
+        store: A MemoryStore, MemorySink or MemorySource object.
+        stix_data (list OR dict OR STIX object): STIX objects to be added
+        allow_custom (bool): Whether to allow custom properties as well unknown
+            custom objects. Note that unknown custom objects cannot be parsed
+            into STIX objects, and will be returned as is. Default: False.
+        version (str): Which STIX2 version to lock the parser to. (e.g. "2.0",
+            "2.1"). If None, the library makes the best effort to figure
+            out the spec representation of the object.
+
+    """
+    if isinstance(stix_data, list):
+        # STIX objects are in a list- recurse on each object
+        for stix_obj in stix_data:
+            _add(store, stix_obj, allow_custom, version)
+
+    elif stix_data["type"] == "bundle":
+        # adding a json bundle - so just grab STIX objects
+        for stix_obj in stix_data.get("objects", []):
+            _add(store, stix_obj, allow_custom, version)
+
+    else:
+        # Adding a single non-bundle object
+        if isinstance(stix_data, _STIXBase):
+            stix_obj = stix_data
+        else:
+            stix_obj = parse(stix_data, allow_custom, version)
+
+        sql_binding_tuples = generate_insert_for_object(stix_obj, isinstance(stix_obj, _Observable))
+        for (sql, bindings) in sql_binding_tuples:
+            store.database_connection.execute(sql, bindings)
+
+
+class RelationalDBSink(DataSink):
+    """Interface for adding/pushing STIX objects to an in-memory dictionary.
+
+    Designed to be paired with a MemorySource, together as the two
+    components of a MemoryStore.
+
+    Args:
+        stix_data (dict OR list): valid STIX 2.0 content in
+            bundle or a list.
+        _store (bool): whether the MemorySink is a part of a MemoryStore,
+            in which case "stix_data" is a direct reference to
+            shared memory with DataSource. Not user supplied
+        allow_custom (bool): whether to allow custom objects/properties
+            when exporting STIX content to file.
+            Default: True.
+        version (str): If present, it forces the parser to use the version
+            provided. Otherwise, the library will make the best effort based
+            on checking the "spec_version" property.
+
+    Attributes:
+        _data (dict): the in-memory dict that holds STIX objects.
+            If part of a MemoryStore, the dict is shared with a MemorySource
+
+    """
+    def __init__(self, database_connection, stix_data=None, allow_custom=True, version=None, _store=False):
+        super(RelationalDBSink, self).__init__()
+        self.allow_custom = allow_custom
+
+        self.database_connection = database_connection
+
+        if _store:
+            self._data = stix_data
+        else:
+            self._data = {}
+            if stix_data:
+                _add(self, stix_data, allow_custom, version)
+
+    def add(self, stix_data, version=None):
+        _add(self, stix_data, self.allow_custom, version)
+    add.__doc__ = _add.__doc__
+
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
new file mode 100644
index 00000000..7f06365d
--- /dev/null
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -0,0 +1,112 @@
+import datetime as dt
+import pytz
+
+import stix2
+from stix2.datastore.relational_db.relational_db import RelationalDBSink
+from stix2.datastore.relational_db.postgres_database_connection import PostgresDatabaseConnection
+from stix2.datastore.relational_db.sql_bindings_creation import generate_insert_for_object
+
+
+directory_stix_object = stix2.Directory(
+    path="/foo/bar/a",
+    path_enc="latin1",
+    ctime="1980-02-23T05:43:28.2678Z",
+    atime="1991-06-09T18:06:33.915Z",
+    mtime="2000-06-28T13:06:09.5827Z",
+    contains_refs=[
+        "file--8903b558-40e3-43e2-be90-b341c12ff7ae",
+        "directory--e0604d0c-bab3-4487-b350-87ac1a3a195c"
+    ],
+    object_marking_refs=[
+        "marking-definition--1b3eec29-5376-4837-bd93-73203e65d73c"
+    ]
+)
+
+s = stix2.v21.Software(
+        name="Word",
+        cpe="cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
+        swid="com.acme.rms-ce-v4-1-5-0",
+        version="2002",
+        languages=["c", "lisp"],
+        vendor="Microsoft",
+    )
+
+
+def windows_registry_key_example():
+    v1 = stix2.v21.WindowsRegistryValueType(
+        name="Foo",
+        data="qwerty",
+        data_type="REG_SZ",
+    )
+    v2 = stix2.v21.WindowsRegistryValueType(
+        name="Bar",
+        data="Fred",
+        data_type="REG_SZ",
+    )
+    w = stix2.v21.WindowsRegistryKey(
+        key="hkey_local_machine\\system\\bar\\foo",
+        values=[v1, v2],
+    )
+    return w
+
+
+def malware_with_all_required_properties():
+
+    ref = stix2.v21.ExternalReference(
+        source_name="veris",
+        external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7",
+        # hashes={
+        #    "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b",
+        #},
+        url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json",
+    )
+    now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
+
+    malware = stix2.v21.Malware(
+        external_references=[ref],
+        type="malware",
+        id="malware--9c4638ec-f1de-4ddb-abf4-1b760417654e",
+        created=now,
+        modified=now,
+        name="Cryptolocker",
+        is_family=False,
+        labels=["foo", "bar"]
+    )
+    return malware
+
+
+def file_example_with_PDFExt_Object():
+    f = stix2.v21.File(
+        name="qwerty.dll",
+        extensions={
+            "pdf-ext": stix2.v21.PDFExt(
+                version="1.7",
+                document_info_dict={
+                    "Title": "Sample document",
+                    "Author": "Adobe Systems Incorporated",
+                    "Creator": "Adobe FrameMaker 5.5.3 for Power Macintosh",
+                    "Producer": "Acrobat Distiller 3.01 for Power Macintosh",
+                    "CreationDate": "20070412090123-02",
+                },
+                pdfid0="DFCE52BD827ECF765649852119D",
+                pdfid1="57A1E0F9ED2AE523E313C",
+            ),
+        },
+    )
+    return f
+
+
+def main():
+    sink = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
+    sink.add(directory_stix_object)
+    sink.add(s)
+    reg_key = windows_registry_key_example()
+    sink.add(reg_key)
+    f = file_example_with_PDFExt_Object()
+    sink.add(f)
+    mal = malware_with_all_required_properties()
+    sink.add(mal)
+
+
+if __name__ == '__main__':
+    main()
\ No newline at end of file
diff --git a/stix2/datastore/relational_db/sql_bindings_creation.py b/stix2/datastore/relational_db/sql_bindings_creation.py
new file mode 100644
index 00000000..6df073d5
--- /dev/null
+++ b/stix2/datastore/relational_db/sql_bindings_creation.py
@@ -0,0 +1,403 @@
+from collections import OrderedDict
+
+from stix2.properties import (
+    DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    IntegerProperty, ListProperty, ReferenceProperty, StringProperty)
+
+# Helps us know which data goes in core, and which in a type-specific table.
+SCO_COMMON_PROPERTIES = {
+    "id",
+    "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged"
+}
+
+# Helps us know which data goes in core, and which in a type-specific table.
+SDO_COMMON_PROPERTIES = {
+    "id",
+    "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged",
+    "created",
+    "modified",
+    "created_by_ref",
+    "revoked",
+    "labels",
+    "confidence",
+    "lang",
+    "external_references"
+}
+
+
+def canonicalize_table_name(schema_name, table_name):
+    full_name = schema_name + "." + table_name
+    return full_name.replace("-", "_")
+
+
+def single_value(p):
+    return not(isinstance(p, (EmbeddedObjectProperty,
+                              ListProperty,
+                              DictionaryProperty)))
+
+
+def table_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
+    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
+        return True
+    else:
+        return False
+
+
+def embedded_object_list_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return isinstance(contained_property, EmbeddedObjectProperty)
+    else:
+        return False
+
+
+def array_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty, EnumProperty))
+    else:
+        return False
+
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def generate_insert_for_array_in_table(type_name, property_name, values, prop, foreign_key_value):
+    table_name = canonicalize_table_name(type_name, property_name)
+    bindings = {
+        "id": foreign_key_value
+    }
+    all_rows_placeholders = []
+
+    for idx, item in enumerate(values):
+        item_binding_name = f"item{idx}"
+
+        all_rows_placeholders.append([
+            f"%(id)s",
+            f"%({item_binding_name})s"
+        ])
+
+        bindings[item_binding_name] = item
+
+    all_rows_sql = ", ".join(
+        "(" + ", ".join(row_placeholders) + ")"
+        for row_placeholders in all_rows_placeholders
+    )
+
+    sql = f"INSERT INTO {table_name}" \
+        f" (id, {derive_column_name(prop)})" \
+        f" VALUES {all_rows_sql}"
+
+    print("ARRAY sql (", table_name, "): ", sql, sep="")
+    print("table array:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
+    bindings, values = generate_single_values(item, item._properties)
+    bindings["id"] = foreign_key_value
+    sql = f"INSERT INTO {canonicalize_table_name(type_name, item._type)}" \
+          f" ({','.join(bindings.keys())})" \
+          f" VALUES ({','.join(values)}, %(id)s )"
+
+    print("sql:", sql)
+    print("embedded:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_dictionary_object(type_name, item, prop, foreign_key_value):
+    values = []
+    bindings = {"id": foreign_key_value}
+    dict_placeholder_rows = []
+
+    for idx, (name, value) in enumerate(item.items()):
+        name_binding = f"name{idx}"
+        value_binding = f"value{idx}"
+
+        dict_placeholder_rows.append([
+            "%(id)s",
+            f"%({name_binding})s",
+            f"%({value_binding})s"
+        ])
+
+        bindings[name_binding] = name
+        bindings[value_binding] = value
+
+    all_rows_sql = ", ".join(
+        "(" + ", ".join(row_placeholders) + ")"
+        for row_placeholders in dict_placeholder_rows
+    )
+
+    sql = f"INSERT INTO {canonicalize_table_name(type_name, prop)}" \
+          " (id, name, value)" \
+          f" VALUES {all_rows_sql}"
+
+    print("sql:", sql)
+
+    print("sql:", sql)
+    print("dict:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_embedded_objects(type_name, values, foreign_key_value):
+    sql_bindings_tuples = list()
+    for item in values:
+        sql_bindings_tuples.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
+    return sql_bindings_tuples
+
+def generate_insert_for_external_references(values, foreign_key_value):
+    sql_bindings_tuples = list()
+    for er in values:
+        bindings = {"id": foreign_key_value}
+        values = []
+        for prop in [ "source_name", "description", "url", "external_id"]:
+            if prop in er:
+                bindings[prop] = er[prop]
+                values.append(f"%({prop})s")
+
+        sql = f"INSERT INTO common.external_reference" \
+              f" ({','.join(bindings.keys())})" \
+              f" VALUES (%(id)s, {','.join(values)})"
+
+        print("sql:", sql)
+        print("er:", bindings)
+        sql_bindings_tuples.append((sql, bindings))
+        if "hashes" in er:
+            sql_bindings_tuples.extend(generate_insert_for_hashes(er["hashes"],
+                                                                  "common.hashes",
+                                                                  foreign_key_value))
+    return sql_bindings_tuples
+
+
+
+def generate_single_values(stix_object, properties, core_properties=[]):
+    values = []
+    bindings = OrderedDict()
+    for name, prop in properties.items():
+        if (single_value(prop) and (name == 'id' or name not in core_properties) or
+                array_property(prop, name, core_properties)):
+            if name in stix_object:
+                bindings[name] = stix_object[name] if not array_property(prop, name, core_properties) else "{" + ",".join(
+                    ['"' + x + '"' for x in stix_object[name]]) + "}"
+                values.append(f"%({name})s")
+    return bindings, values
+
+
+def generate_insert_for_object(stix_object, stix_object_sco):
+    sql_bindings_tuples = list()
+    if stix_object_sco:
+        core_properties = SCO_COMMON_PROPERTIES
+    else:
+        core_properties = SDO_COMMON_PROPERTIES
+    type_name = stix_object["type"]
+    table_name = canonicalize_table_name(type_name, type_name)
+    properties = stix_object._properties
+    sql_bindings_tuples.extend(generate_insert_for_core(stix_object, core_properties))
+
+    bindings, values = generate_single_values(stix_object, properties, core_properties)
+
+    sql = f"INSERT INTO {table_name}" \
+        f" ({','.join(bindings.keys())})" \
+        f" VALUES ({','.join(values)})"
+    sql_bindings_tuples.append((sql, bindings))
+
+    print("sql:", sql)
+    print("obj:", bindings)
+
+    if "external_references" in stix_object:
+        sql_bindings_tuples.extend(generate_insert_for_external_references(stix_object["external_references"],
+                                                                           stix_object["id"]))
+
+    for name, prop in properties.items():
+        if table_property(prop, name, core_properties):
+            if name in stix_object:
+                if embedded_object_list_property(prop, name, core_properties):
+                    sql_bindings_tuples.extend(generate_insert_for_embedded_objects(stix_object["type"],
+                                                                                    stix_object[name],
+                                                                                    stix_object["id"]))
+                elif isinstance(prop, ExtensionsProperty):
+                    pass
+                else:
+                    sql_bindings_tuples.extend(generate_insert_for_array_in_table(stix_object["type"],
+                                                                                  name,
+                                                                                  stix_object[name],
+                                                                                  properties[name],
+                                                                                  stix_object["id"] ))
+    return sql_bindings_tuples
+
+
+def generate_insert_for_hashes(hashes, hashes_table_name, **foreign_key):
+    foreign_key_name, foreign_key_value = next(iter(foreign_key.items()))
+
+    bindings = {
+        foreign_key_name: foreign_key_value
+    }
+
+    all_rows_placeholders = []
+    for idx, (hash_name, hash_value) in enumerate(hashes.items()):
+        hash_name_binding_name = "hash_name" + str(idx)
+        hash_value_binding_name = "hash_value" + str(idx)
+
+        all_rows_placeholders.append([
+            f"%({foreign_key_name})s",
+            f"%({hash_name_binding_name})s",
+            f"%({hash_value_binding_name})s"
+        ])
+
+        bindings[hash_name_binding_name] = hash_name
+        bindings[hash_value_binding_name] = hash_value
+
+    all_rows_placeholders_sql = ", ".join(
+        "(" + ", ".join(row_placeholders) + ")"
+        for row_placeholders in all_rows_placeholders
+    )
+
+    sql = f"INSERT INTO {hashes_table_name}" \
+          f"({foreign_key_name}, hash_name, hash_value)" \
+          f" VALUES {all_rows_placeholders_sql}"
+
+    print("HASHES sql (", hashes_table_name, "): ", sql, sep="")
+    print("Hashes:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_granular_markings(granular_markings, markings_table_name, **foreign_key):
+    foreign_key_column, foreign_key_value = next(iter(foreign_key.items()))
+
+    bindings = {
+        foreign_key_column: foreign_key_value
+    }
+
+    all_rows_placeholders = []
+    for idx, granular_marking in enumerate(granular_markings):
+        lang_binding_name = f"lang{idx}"
+        marking_ref_binding_name = f"marking_ref{idx}"
+        selectors_binding_name = f"selectors{idx}"
+
+        all_rows_placeholders.append([
+            f"%({foreign_key_column})s",
+            f"%({lang_binding_name})s",
+            f"%({marking_ref_binding_name})s",
+            f"%({selectors_binding_name})s"
+        ])
+
+        bindings[lang_binding_name] = granular_marking.get("lang")
+        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
+        bindings[selectors_binding_name] = granular_marking.get("selectors")
+
+    all_rows_placeholders_sql = ", ".join(
+        "(" + ", ".join(row_placeholders) + ")"
+        for row_placeholders in all_rows_placeholders
+    )
+
+    sql = f'INSERT INTO {markings_table_name}' \
+          f" ({foreign_key_column}, lang, marking_ref, selectors)" \
+          f" VALUES {all_rows_placeholders_sql}"
+
+    print("GRANULAR MARKINGS sql (", markings_table_name, "): ", sql, sep="")
+    print("granular:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
+    sql_bindings_tuples = list()
+    for name, ex in extensions.items():
+        sql_bindings_tuples.extend(generate_insert_for_subtype_extension(name,
+                                                                         ex,
+                                                                         foreign_key_value,
+                                                                         type_name,
+                                                                         core_properties))
+    return sql_bindings_tuples
+
+
+def generate_insert_for_subtype_extension(name, ex, foreign_key_value, type_name, core_properties):
+    sql_bindings_tuples = list()
+    properties = ex._properties
+
+    bindings, values = generate_single_values(ex, properties, core_properties)
+    bindings["id"] = foreign_key_value
+    sql = f"INSERT INTO {canonicalize_table_name(type_name, name)}" \
+          f" ({','.join(bindings.keys())})" \
+          f" VALUES ({','.join(values)}, %(id)s)"
+
+    print("sql:", sql)
+    print("ext:", bindings)
+    sql_bindings_tuples.append((sql, bindings))
+    if "external_references" in ex:
+        sql_bindings_tuples.extend(generate_insert_for_external_references(ex["external_references"], ex["id"]))
+
+    for name, prop in properties.items():
+        if table_property(prop, name, core_properties):
+            if name in ex:
+                if embedded_object_list_property(prop, name, core_properties):
+                    sql_bindings_tuples.extend(generate_insert_for_embedded_objects(ex["type"], ex[name], ex["id"]))
+                elif isinstance(prop, DictionaryProperty) and name not in core_properties:
+                    sql_bindings_tuples.extend(generate_insert_for_dictionary_object(type_name, ex[name], name, foreign_key_value))
+                else:
+                    sql_bindings_tuples.extend(generate_insert_for_array_in_table(ex["type"], name, ex[name], properties[name],
+                                                       ex["id"]))
+    return sql_bindings_tuples
+
+
+def generate_insert_for_core(stix_object, core_properties):
+    kind_of_stix_object = "sdo" if "created" in core_properties else "sco"
+    sql_bindings_tuples = list()
+    core_bindings = {}
+
+    for prop_name, value in stix_object.items():
+
+        if prop_name in core_properties:
+            # stored in separate tables, skip here
+            if prop_name not in {
+                "object_marking_refs", "granular_markings", "external_references"
+            }:
+                core_bindings[prop_name] = value
+
+    core_col_names = ", ".join(core_bindings)
+    core_placeholders = ", ".join(
+        f"%({name})s" for name in core_bindings
+    )
+    sql = f"INSERT INTO common.core_{kind_of_stix_object} ({core_col_names}) VALUES ({core_placeholders})"
+    print("CORE sql:", sql)
+    print(core_bindings)
+    sql_bindings_tuples.append((sql, core_bindings))
+
+    if "object_marking_refs" in stix_object:
+        sql_bindings_tuples.extend(generate_insert_for_array_in_table(
+            "core",
+            "object_marking_refs",
+            stix_object.object_marking_refs,
+            stix_object._properties["object_marking_refs"],
+            stix_object.id
+        ))
+
+    # Granular markings
+    if "granular_markings" in stix_object:
+        sql_bindings_tuples.extend(generate_insert_for_granular_markings(
+            stix_object.granular_markings, "common.granular_marking_stix_object",
+            core_id=stix_object.id
+        ))
+
+    if "extensions" in stix_object:
+        sql_bindings_tuples.extend(generate_insert_for_extensions(stix_object.extensions,
+                                                                  stix_object.id,
+                                                                  stix_object.type,
+                                                                  core_properties))
+    return sql_bindings_tuples
+

From 281c81bc4d71f92922de23027f4e9f6e51984aae Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 14 Mar 2024 15:54:58 -0400
Subject: [PATCH 002/132] added create_insert_statement to DatabaseConnection

---
 stix2/datastore/relational_db/__init__.py      | 18 +++++++++++++++++-
 .../postgres_database_connection.py            |  3 +++
 stix2/datastore/relational_db/relational_db.py |  4 +++-
 .../relational_db/sql_bindings_creation.py     |  9 +++++----
 4 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/stix2/datastore/relational_db/__init__.py b/stix2/datastore/relational_db/__init__.py
index 1dcd772e..c0dc714c 100644
--- a/stix2/datastore/relational_db/__init__.py
+++ b/stix2/datastore/relational_db/__init__.py
@@ -11,8 +11,24 @@ def execute(self, sql_statement, bindings):
         """
 
         Args:
-            bindings:
+            sql_statement: the statement to execute
+            bindings: a dictionary where the keys are the column names and the values are the data to be
+            inserted into that column of the table
 
         Returns:
 
         """
+
+    @abstractmethod
+    def create_insert_statement (self, table_name, bindings, **kwargs):
+        """
+
+        Args:
+            table_name: the name of the table to be inserted into
+            bindings: a dictionary where the keys are the column names and the values are the data to be
+            inserted into that column of the table
+            **kwargs: other specific arguments
+
+        Returns:
+
+        """
\ No newline at end of file
diff --git a/stix2/datastore/relational_db/postgres_database_connection.py b/stix2/datastore/relational_db/postgres_database_connection.py
index 3c92a058..9654c84f 100644
--- a/stix2/datastore/relational_db/postgres_database_connection.py
+++ b/stix2/datastore/relational_db/postgres_database_connection.py
@@ -10,3 +10,6 @@ def __init__(self, host, dbname, user):
 
     def execute(self, sql_statement, bindings):
         self.db.run(sql_statement, parameters=bindings)
+
+    def create_insert_statement(self, table_name, bindings, **kwargs):
+        return f"INSERT INTO {table_name} ({','.join(bindings.keys())}) VALUES ({','.join(kwargs['values'])})"
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 28f0191d..68bcfc9f 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -39,7 +39,9 @@ def _add(store, stix_data, allow_custom=True, version=None):
         else:
             stix_obj = parse(stix_data, allow_custom, version)
 
-        sql_binding_tuples = generate_insert_for_object(stix_obj, isinstance(stix_obj, _Observable))
+        sql_binding_tuples = generate_insert_for_object(store.database_connection,
+                                                        stix_obj,
+                                                        isinstance(stix_obj, _Observable))
         for (sql, bindings) in sql_binding_tuples:
             store.database_connection.execute(sql, bindings)
 
diff --git a/stix2/datastore/relational_db/sql_bindings_creation.py b/stix2/datastore/relational_db/sql_bindings_creation.py
index 6df073d5..49d44a25 100644
--- a/stix2/datastore/relational_db/sql_bindings_creation.py
+++ b/stix2/datastore/relational_db/sql_bindings_creation.py
@@ -199,7 +199,7 @@ def generate_single_values(stix_object, properties, core_properties=[]):
     return bindings, values
 
 
-def generate_insert_for_object(stix_object, stix_object_sco):
+def generate_insert_for_object(database_connection, stix_object, stix_object_sco):
     sql_bindings_tuples = list()
     if stix_object_sco:
         core_properties = SCO_COMMON_PROPERTIES
@@ -212,9 +212,10 @@ def generate_insert_for_object(stix_object, stix_object_sco):
 
     bindings, values = generate_single_values(stix_object, properties, core_properties)
 
-    sql = f"INSERT INTO {table_name}" \
-        f" ({','.join(bindings.keys())})" \
-        f" VALUES ({','.join(values)})"
+    # sql = f"INSERT INTO {table_name}" \
+    #     f" ({','.join(bindings.keys())})" \
+    #     f" VALUES ({','.join(values)})"
+    sql = database_connection.create_insert_statement(table_name, bindings, values=values)
     sql_bindings_tuples.append((sql, bindings))
 
     print("sql:", sql)

From 89c21535f3f44bfbff962068a4f95aec5702fb01 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 17 Mar 2024 16:39:42 -0400
Subject: [PATCH 003/132] create the tables

---
 .../relational_db/relational_db_testing.py    |  33 +-
 .../datastore/relational_db/table_creation.py | 405 ++++++++++++++++++
 2 files changed, 426 insertions(+), 12 deletions(-)
 create mode 100644 stix2/datastore/relational_db/table_creation.py

diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 7f06365d..d033005b 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -2,9 +2,11 @@
 import pytz
 
 import stix2
+
+from stix2.v21.base import _DomainObject, _Observable, _Extension
 from stix2.datastore.relational_db.relational_db import RelationalDBSink
 from stix2.datastore.relational_db.postgres_database_connection import PostgresDatabaseConnection
-from stix2.datastore.relational_db.sql_bindings_creation import generate_insert_for_object
+from stix2.datastore.relational_db.table_creation import generate_object_table, get_all_subclasses, create_core_sdo_table
 
 
 directory_stix_object = stix2.Directory(
@@ -51,7 +53,6 @@ def windows_registry_key_example():
 
 
 def malware_with_all_required_properties():
-
     ref = stix2.v21.ExternalReference(
         source_name="veris",
         external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7",
@@ -95,17 +96,25 @@ def file_example_with_PDFExt_Object():
     )
     return f
 
-
 def main():
-    sink = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
-    sink.add(directory_stix_object)
-    sink.add(s)
-    reg_key = windows_registry_key_example()
-    sink.add(reg_key)
-    f = file_example_with_PDFExt_Object()
-    sink.add(f)
-    mal = malware_with_all_required_properties()
-    sink.add(mal)
+    # sink = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
+    # sink.add(directory_stix_object)
+    # sink.add(s)
+    # reg_key = windows_registry_key_example()
+    # sink.add(reg_key)
+    # f = file_example_with_PDFExt_Object()
+    # sink.add(f)
+    # mal = malware_with_all_required_properties()
+    # sink.add(mal)
+
+    create_core_sdo_table()
+    for sdo_class in get_all_subclasses(_DomainObject):
+        x = generate_object_table(sdo_class)
+    for sdo_class in get_all_subclasses(_Observable):
+        x = generate_object_table(sdo_class)
+    for sdo_class in get_all_subclasses(_Extension):
+        x = generate_object_table(sdo_class, is_extension=True)
+
 
 
 if __name__ == '__main__':
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
new file mode 100644
index 00000000..a8df7f4d
--- /dev/null
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -0,0 +1,405 @@
+from collections import OrderedDict
+
+import inspect
+import re
+
+from sqlalchemy import (
+    ARRAY,
+    SERIES,
+    TIMESTAMP,
+    Boolean,
+    CheckConstraint,
+    Column,
+    Float,
+    ForeignKey,
+    Integer,
+    LargeBinary,
+    MetaData,
+    Table,
+    Text,
+    create_engine,
+    insert,
+)
+from sqlalchemy.schema import CreateTable
+
+metadata = MetaData()
+
+
+from stix2.properties import (
+    Property, BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
+    ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty,
+    ListProperty, ObjectReferenceProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty)
+from stix2.v21.common import KillChainPhase
+
+from stix2.v21.base import _STIXBase21
+from stix2.v21.base import _DomainObject
+
+# Helps us know which data goes in core, and which in a type-specific table.
+SCO_COMMON_PROPERTIES = {
+    "id",
+    # "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged"
+}
+
+# Helps us know which data goes in core, and which in a type-specific table.
+SDO_COMMON_PROPERTIES = {
+    "id",
+    # "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged",
+    "created",
+    "modified",
+    "created_by_ref",
+    "revoked",
+    "labels",
+    "confidence",
+    "lang",
+    "external_references"
+}
+
+
+def canonicalize_table_name_with_schema(schema_name, table_name):
+    full_name = schema_name + "." + table_name
+    return full_name.replace("-", "_")
+
+def canonicalize_table_name(table_name):
+    return table_name.replace("-", "_")
+
+
+def aux_table_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
+    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
+        return True
+    else:
+        return False
+
+
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def get_all_subclasses(cls):
+    all_subclasses = []
+
+    for subclass in cls.__subclasses__():
+        all_subclasses.append(subclass)
+        all_subclasses.extend(get_all_subclasses(subclass))
+
+    return all_subclasses
+
+def create_core_sdo_table():
+    x = Table("core_sdo",
+        metadata,
+        Column(
+            "id",
+            Text,
+            CheckConstraint(
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
+            ),
+            primary_key=True,
+        ),
+        Column("spec_version", Text, default="2.1"),
+        Column("created_by_ref",
+               Text,
+               CheckConstraint(
+                   "created_by_ref ~ ^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$"
+                )),
+        Column("created", TIMESTAMP(timezone=True)),
+        Column("modified", TIMESTAMP(timezone=True)),
+        Column("revoked", Boolean),
+        Column("confidence", Integer),
+        Column("lang", Text),
+        Column("object_marking_ref", ARRAY(Text)),
+        schema="common"
+    )
+    print(CreateTable(x))
+
+
+# _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
+#
+#
+# _ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
+
+
+def create_real_method_name(name, klass_name):
+    # if klass_name not in _ALLOWABLE_CLASSES:
+    #     raise NameError
+    # split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
+    # split_up_klass_name.remove("Type")
+    return name + "_" + "_".join([x.lower() for x in klass_name])
+
+
+def add_method(cls):
+    def decorator(fn):
+        method_name = fn.__name__
+        fn.__name__ = create_real_method_name(fn.__name__, cls.__name__)
+        setattr(cls, method_name, fn)
+        return fn
+    return decorator
+
+
+@add_method(KillChainPhase)
+def determine_sql_type(self):
+    return None
+
+
+@add_method(Property)
+def generate_table_information(self, name, is_sdo, table_name):
+    pass
+
+
+@add_method(Property)
+def determine_sql_type(self):
+    pass
+
+
+@add_method(StringProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  Text,
+                  nullable=not(self.required),
+                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+
+
+@add_method(StringProperty)
+def determine_sql_type(self):
+    return Text
+
+
+@add_method(IntegerProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  Integer,
+                  nullable=not(self.required),
+                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+
+
+@add_method(IntegerProperty)
+def determine_sql_type(self):
+    return Integer
+
+@add_method(FloatProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  Float,
+                  nullable=not(self.required),
+                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+
+@add_method(FloatProperty)
+def determine_sql_type(self):
+    return Float
+
+@add_method(BooleanProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  Boolean,
+                  nullable=not(self.required),
+                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+
+
+@add_method(BooleanProperty)
+def determine_sql_type(self):
+    return Boolean
+
+
+@add_method(TypeProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  Text,
+                  nullable=not(self.required),
+                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+
+
+@add_method(IDProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    foreign_key_column = "common.core_sdo.id" if is_sdo else "common.core_sco.id"
+    return Column(name,
+                  Text,
+                  ForeignKey(foreign_key_column, ondelete="CASCADE"),
+                  CheckConstraint(
+                      f"{name} ~ '^{table_name}--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
+                  ),
+                  primary_key=True,
+                  nullable=not (self.required))
+
+@add_method(EnumProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    enum_re = "|".join(self.allowed)
+    return Column(name,
+                  Text,
+                  CheckConstraint(
+                      f"{name} ~ '^{enum_re}$'"
+                  ),
+                  nullable=not (self.required))
+
+
+@add_method(TimestampProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  TIMESTAMP(timezone=True),
+                  # CheckConstraint(
+                  #     f"{name} ~ '^{enum_re}$'"
+                  # ),
+                  nullable=not (self.required))
+
+@add_method(DictionaryProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    columns = list()
+    columns.append(Column("id",
+                          Text,
+                          ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE")))
+    columns.append(Column("name",
+                          Text,
+                          nullable=False))
+    columns.append(Column("value",
+                          Text,
+                          nullable=False))
+    return Table(canonicalize_table_name(name), metadata, *columns)
+
+@add_method(HashesProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+
+    columns = list()
+    columns.append(Column("id",
+                          Text,
+                          ForeignKey(canonicalize_table_name(table_name),
+                                     ondelete="CASCADE")))
+    columns.append(Column("hash_name",
+                          Text,
+                          nullable=False))
+    columns.append(Column("hash_value",
+                          Text,
+                          nullable=False))
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+
+@add_method(HexProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return Column(name,
+                  LargeBinary,
+                  nullable=not (self.required))
+
+@add_method(BinaryProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    print("BinaryProperty not handled, yet")
+    return None
+
+@add_method(ExtensionsProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    columns = list()
+    columns.append(Column("id",
+                          Text,
+                          ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE")))
+    columns.append(Column("ext_table_name",
+                          Text,
+                          nullable=False))
+    columns.append(Column("ext_table_id",
+                          Text,
+                          nullable=False))
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+
+
+def ref_column(name, specifics):
+    if specifics:
+        allowed_types = "|".join(specifics)
+        return Column(name,
+                      Text,
+                      CheckConstraint(
+                          f"{name} ~ '^({allowed_types})--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
+                      ))
+    else:
+        return Column(name,
+                      Text,
+                      nullable=False)
+
+
+@add_method(ReferenceProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return ref_column(name, self.specifics)
+
+
+@add_method(EmbeddedObjectProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    return generate_object_table(self.type, table_name)
+
+
+@add_method(ObjectReferenceProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
+
+
+@add_method(ListProperty)
+def generate_table_information(self, name, is_sdo, table_name):
+    if isinstance(self.contained, ReferenceProperty):
+        columns = list()
+        columns.append(Column("id",
+                              Text,
+                              ForeignKey(canonicalize_table_name(table_name),
+                                         ondelete="CASCADE")))
+        columns.append(ref_column("ref_id", self.contained.specifics))
+        return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+    elif isinstance(self.contained, EmbeddedObjectProperty):
+        columns = list()
+        columns.append(Column("id",
+                              Text,
+                              ForeignKey(canonicalize_table_name(table_name),
+                                         ondelete="CASCADE")))
+        columns.append(Column("ref_id",
+                              Text,
+                              nullable=False))
+        CreateTable(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns))
+        return self.contained.generate_table_information(name, False, canonicalize_table_name(table_name + "_" + name))
+    else:
+        if isinstance(self.contained, Property):
+            sql_type = self.contained.determine_sql_type()
+            if sql_type:
+                return Column(name,
+                              ARRAY(sql_type),
+                              nullable=not (self.required))
+
+
+def generate_object_table(stix_object_class, foreign_key_name=None, is_extension=False):
+    properties = stix_object_class._properties
+    if hasattr(stix_object_class, "_type"):
+        table_name = stix_object_class._type
+    else:
+        table_name = stix_object_class.__name__
+    is_sdo = True # isinstance(stix_object_class, _DomainObject)
+    core_properties = SDO_COMMON_PROPERTIES if is_sdo else SCO_COMMON_PROPERTIES
+    columns = list()
+    tables = list()
+    for name, prop in properties.items():
+        if name == 'id' or name not in core_properties :
+                # or array_property(prop, name, core_properties)):
+                col = prop.generate_table_information(name, is_sdo, table_name)
+                if col is not None and isinstance(col, Column):
+                    columns.append(col)
+                if col is not None and isinstance(col, Table):
+                    tables.append(col)
+    if is_extension:
+        columns.append(Column("id", SERIES, primary_key=True))
+    if foreign_key_name:
+         columns.append(Column("id",
+                               Text,
+                              ForeignKey(canonicalize_table_name(foreign_key_name),
+                                          ondelete="CASCADE")))
+         return Table(canonicalize_table_name(table_name), metadata, *columns)
+    else:
+        x = Table(canonicalize_table_name(table_name), metadata, *columns)
+        print(CreateTable(x))
+    for t in tables:
+        print(CreateTable(t))
+
+
+

From 4a12d7d6d98b91f05539672198178bcf0a68340f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 18 Mar 2024 12:56:31 -0400
Subject: [PATCH 004/132] fix dictionary table generation, remove SERIES

---
 stix2/datastore/relational_db/table_creation.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index a8df7f4d..b1551d64 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -5,7 +5,6 @@
 
 from sqlalchemy import (
     ARRAY,
-    SERIES,
     TIMESTAMP,
     Boolean,
     CheckConstraint,
@@ -266,7 +265,7 @@ def generate_table_information(self, name, is_sdo, table_name):
     columns.append(Column("value",
                           Text,
                           nullable=False))
-    return Table(canonicalize_table_name(name), metadata, *columns)
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
 
 @add_method(HashesProperty)
 def generate_table_information(self, name, is_sdo, table_name):
@@ -388,7 +387,7 @@ def generate_object_table(stix_object_class, foreign_key_name=None, is_extension
                 if col is not None and isinstance(col, Table):
                     tables.append(col)
     if is_extension:
-        columns.append(Column("id", SERIES, primary_key=True))
+        columns.append(Column("id", primary_key=True))
     if foreign_key_name:
          columns.append(Column("id",
                                Text,

From 73805d3f75c7c0fa837b6b1ea4dde1f012120744 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 18 Mar 2024 13:58:38 -0400
Subject: [PATCH 005/132] flakey

---
 stix2/datastore/relational_db/__init__.py     |   7 +-
 .../datastore/relational_db/relational_db.py  |  16 +-
 .../relational_db/relational_db_testing.py    |  34 +-
 .../relational_db/sql_bindings_creation.py    | 404 -----------------
 .../datastore/relational_db/table_creation.py | 421 +++++++++++-------
 5 files changed, 281 insertions(+), 601 deletions(-)
 delete mode 100644 stix2/datastore/relational_db/sql_bindings_creation.py

diff --git a/stix2/datastore/relational_db/__init__.py b/stix2/datastore/relational_db/__init__.py
index c0dc714c..1adb8f10 100644
--- a/stix2/datastore/relational_db/__init__.py
+++ b/stix2/datastore/relational_db/__init__.py
@@ -2,7 +2,6 @@
 
 
 class DatabaseConnection():
-
     def __init__(self):
         pass
 
@@ -18,9 +17,10 @@ def execute(self, sql_statement, bindings):
         Returns:
 
         """
+        pass
 
     @abstractmethod
-    def create_insert_statement (self, table_name, bindings, **kwargs):
+    def create_insert_statement(self, table_name, bindings, **kwargs):
         """
 
         Args:
@@ -31,4 +31,5 @@ def create_insert_statement (self, table_name, bindings, **kwargs):
 
         Returns:
 
-        """
\ No newline at end of file
+        """
+        pass
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 68bcfc9f..aeceb4eb 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,8 +1,9 @@
-from stix2 import v20, v21
-from stix2.base import _STIXBase, _Observable
+from stix2.base import _Observable, _STIXBase
 from stix2.datastore import DataSink
+from stix2.datastore.relational_db.sql_bindings_creation import (
+    generate_insert_for_object,
+)
 from stix2.parsing import parse
-from stix2.datastore.relational_db.sql_bindings_creation import generate_insert_for_object
 
 
 def _add(store, stix_data, allow_custom=True, version=None):
@@ -39,9 +40,11 @@ def _add(store, stix_data, allow_custom=True, version=None):
         else:
             stix_obj = parse(stix_data, allow_custom, version)
 
-        sql_binding_tuples = generate_insert_for_object(store.database_connection,
-                                                        stix_obj,
-                                                        isinstance(stix_obj, _Observable))
+        sql_binding_tuples = generate_insert_for_object(
+            store.database_connection,
+            stix_obj,
+            isinstance(stix_obj, _Observable),
+        )
         for (sql, bindings) in sql_binding_tuples:
             store.database_connection.execute(sql, bindings)
 
@@ -86,4 +89,3 @@ def __init__(self, database_connection, stix_data=None, allow_custom=True, versi
     def add(self, stix_data, version=None):
         _add(self, stix_data, self.allow_custom, version)
     add.__doc__ = _add.__doc__
-
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index d033005b..d5fc165e 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -1,13 +1,13 @@
 import datetime as dt
+
 import pytz
 
 import stix2
-
-from stix2.v21.base import _DomainObject, _Observable, _Extension
-from stix2.datastore.relational_db.relational_db import RelationalDBSink
-from stix2.datastore.relational_db.postgres_database_connection import PostgresDatabaseConnection
-from stix2.datastore.relational_db.table_creation import generate_object_table, get_all_subclasses, create_core_sdo_table
-
+# from stix2.datastore.relational_db.relational_db import RelationalDBSink
+from stix2.datastore.relational_db.table_creation import (
+    create_core_sdo_table, generate_object_table, get_all_subclasses,
+)
+from stix2.v21.base import _DomainObject, _Extension, _Observable
 
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
@@ -17,11 +17,11 @@
     mtime="2000-06-28T13:06:09.5827Z",
     contains_refs=[
         "file--8903b558-40e3-43e2-be90-b341c12ff7ae",
-        "directory--e0604d0c-bab3-4487-b350-87ac1a3a195c"
+        "directory--e0604d0c-bab3-4487-b350-87ac1a3a195c",
     ],
     object_marking_refs=[
-        "marking-definition--1b3eec29-5376-4837-bd93-73203e65d73c"
-    ]
+        "marking-definition--1b3eec29-5376-4837-bd93-73203e65d73c",
+    ],
 )
 
 s = stix2.v21.Software(
@@ -31,7 +31,7 @@
         version="2002",
         languages=["c", "lisp"],
         vendor="Microsoft",
-    )
+)
 
 
 def windows_registry_key_example():
@@ -58,7 +58,7 @@ def malware_with_all_required_properties():
         external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7",
         # hashes={
         #    "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b",
-        #},
+        # },
         url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json",
     )
     now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
@@ -71,7 +71,7 @@ def malware_with_all_required_properties():
         modified=now,
         name="Cryptolocker",
         is_family=False,
-        labels=["foo", "bar"]
+        labels=["foo", "bar"],
     )
     return malware
 
@@ -96,6 +96,7 @@ def file_example_with_PDFExt_Object():
     )
     return f
 
+
 def main():
     # sink = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
     # sink.add(directory_stix_object)
@@ -109,13 +110,12 @@ def main():
 
     create_core_sdo_table()
     for sdo_class in get_all_subclasses(_DomainObject):
-        x = generate_object_table(sdo_class)
+        generate_object_table(sdo_class)
     for sdo_class in get_all_subclasses(_Observable):
-        x = generate_object_table(sdo_class)
+        generate_object_table(sdo_class)
     for sdo_class in get_all_subclasses(_Extension):
-        x = generate_object_table(sdo_class, is_extension=True)
-
+        generate_object_table(sdo_class, is_extension=True)
 
 
 if __name__ == '__main__':
-    main()
\ No newline at end of file
+    main()
diff --git a/stix2/datastore/relational_db/sql_bindings_creation.py b/stix2/datastore/relational_db/sql_bindings_creation.py
deleted file mode 100644
index 49d44a25..00000000
--- a/stix2/datastore/relational_db/sql_bindings_creation.py
+++ /dev/null
@@ -1,404 +0,0 @@
-from collections import OrderedDict
-
-from stix2.properties import (
-    DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
-    IntegerProperty, ListProperty, ReferenceProperty, StringProperty)
-
-# Helps us know which data goes in core, and which in a type-specific table.
-SCO_COMMON_PROPERTIES = {
-    "id",
-    "type",
-    "spec_version",
-    "object_marking_refs",
-    "granular_markings",
-    "defanged"
-}
-
-# Helps us know which data goes in core, and which in a type-specific table.
-SDO_COMMON_PROPERTIES = {
-    "id",
-    "type",
-    "spec_version",
-    "object_marking_refs",
-    "granular_markings",
-    "defanged",
-    "created",
-    "modified",
-    "created_by_ref",
-    "revoked",
-    "labels",
-    "confidence",
-    "lang",
-    "external_references"
-}
-
-
-def canonicalize_table_name(schema_name, table_name):
-    full_name = schema_name + "." + table_name
-    return full_name.replace("-", "_")
-
-
-def single_value(p):
-    return not(isinstance(p, (EmbeddedObjectProperty,
-                              ListProperty,
-                              DictionaryProperty)))
-
-
-def table_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
-    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
-        return True
-    else:
-        return False
-
-
-def embedded_object_list_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return isinstance(contained_property, EmbeddedObjectProperty)
-    else:
-        return False
-
-
-def array_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty, EnumProperty))
-    else:
-        return False
-
-
-def derive_column_name(prop):
-    contained_property = prop.contained
-    if isinstance(contained_property, ReferenceProperty):
-        return "ref_id"
-    elif isinstance(contained_property, StringProperty):
-        return "value"
-
-
-def generate_insert_for_array_in_table(type_name, property_name, values, prop, foreign_key_value):
-    table_name = canonicalize_table_name(type_name, property_name)
-    bindings = {
-        "id": foreign_key_value
-    }
-    all_rows_placeholders = []
-
-    for idx, item in enumerate(values):
-        item_binding_name = f"item{idx}"
-
-        all_rows_placeholders.append([
-            f"%(id)s",
-            f"%({item_binding_name})s"
-        ])
-
-        bindings[item_binding_name] = item
-
-    all_rows_sql = ", ".join(
-        "(" + ", ".join(row_placeholders) + ")"
-        for row_placeholders in all_rows_placeholders
-    )
-
-    sql = f"INSERT INTO {table_name}" \
-        f" (id, {derive_column_name(prop)})" \
-        f" VALUES {all_rows_sql}"
-
-    print("ARRAY sql (", table_name, "): ", sql, sep="")
-    print("table array:", bindings)
-    return [(sql, bindings)]
-
-
-def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
-    bindings, values = generate_single_values(item, item._properties)
-    bindings["id"] = foreign_key_value
-    sql = f"INSERT INTO {canonicalize_table_name(type_name, item._type)}" \
-          f" ({','.join(bindings.keys())})" \
-          f" VALUES ({','.join(values)}, %(id)s )"
-
-    print("sql:", sql)
-    print("embedded:", bindings)
-    return [(sql, bindings)]
-
-
-def generate_insert_for_dictionary_object(type_name, item, prop, foreign_key_value):
-    values = []
-    bindings = {"id": foreign_key_value}
-    dict_placeholder_rows = []
-
-    for idx, (name, value) in enumerate(item.items()):
-        name_binding = f"name{idx}"
-        value_binding = f"value{idx}"
-
-        dict_placeholder_rows.append([
-            "%(id)s",
-            f"%({name_binding})s",
-            f"%({value_binding})s"
-        ])
-
-        bindings[name_binding] = name
-        bindings[value_binding] = value
-
-    all_rows_sql = ", ".join(
-        "(" + ", ".join(row_placeholders) + ")"
-        for row_placeholders in dict_placeholder_rows
-    )
-
-    sql = f"INSERT INTO {canonicalize_table_name(type_name, prop)}" \
-          " (id, name, value)" \
-          f" VALUES {all_rows_sql}"
-
-    print("sql:", sql)
-
-    print("sql:", sql)
-    print("dict:", bindings)
-    return [(sql, bindings)]
-
-
-def generate_insert_for_embedded_objects(type_name, values, foreign_key_value):
-    sql_bindings_tuples = list()
-    for item in values:
-        sql_bindings_tuples.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
-    return sql_bindings_tuples
-
-def generate_insert_for_external_references(values, foreign_key_value):
-    sql_bindings_tuples = list()
-    for er in values:
-        bindings = {"id": foreign_key_value}
-        values = []
-        for prop in [ "source_name", "description", "url", "external_id"]:
-            if prop in er:
-                bindings[prop] = er[prop]
-                values.append(f"%({prop})s")
-
-        sql = f"INSERT INTO common.external_reference" \
-              f" ({','.join(bindings.keys())})" \
-              f" VALUES (%(id)s, {','.join(values)})"
-
-        print("sql:", sql)
-        print("er:", bindings)
-        sql_bindings_tuples.append((sql, bindings))
-        if "hashes" in er:
-            sql_bindings_tuples.extend(generate_insert_for_hashes(er["hashes"],
-                                                                  "common.hashes",
-                                                                  foreign_key_value))
-    return sql_bindings_tuples
-
-
-
-def generate_single_values(stix_object, properties, core_properties=[]):
-    values = []
-    bindings = OrderedDict()
-    for name, prop in properties.items():
-        if (single_value(prop) and (name == 'id' or name not in core_properties) or
-                array_property(prop, name, core_properties)):
-            if name in stix_object:
-                bindings[name] = stix_object[name] if not array_property(prop, name, core_properties) else "{" + ",".join(
-                    ['"' + x + '"' for x in stix_object[name]]) + "}"
-                values.append(f"%({name})s")
-    return bindings, values
-
-
-def generate_insert_for_object(database_connection, stix_object, stix_object_sco):
-    sql_bindings_tuples = list()
-    if stix_object_sco:
-        core_properties = SCO_COMMON_PROPERTIES
-    else:
-        core_properties = SDO_COMMON_PROPERTIES
-    type_name = stix_object["type"]
-    table_name = canonicalize_table_name(type_name, type_name)
-    properties = stix_object._properties
-    sql_bindings_tuples.extend(generate_insert_for_core(stix_object, core_properties))
-
-    bindings, values = generate_single_values(stix_object, properties, core_properties)
-
-    # sql = f"INSERT INTO {table_name}" \
-    #     f" ({','.join(bindings.keys())})" \
-    #     f" VALUES ({','.join(values)})"
-    sql = database_connection.create_insert_statement(table_name, bindings, values=values)
-    sql_bindings_tuples.append((sql, bindings))
-
-    print("sql:", sql)
-    print("obj:", bindings)
-
-    if "external_references" in stix_object:
-        sql_bindings_tuples.extend(generate_insert_for_external_references(stix_object["external_references"],
-                                                                           stix_object["id"]))
-
-    for name, prop in properties.items():
-        if table_property(prop, name, core_properties):
-            if name in stix_object:
-                if embedded_object_list_property(prop, name, core_properties):
-                    sql_bindings_tuples.extend(generate_insert_for_embedded_objects(stix_object["type"],
-                                                                                    stix_object[name],
-                                                                                    stix_object["id"]))
-                elif isinstance(prop, ExtensionsProperty):
-                    pass
-                else:
-                    sql_bindings_tuples.extend(generate_insert_for_array_in_table(stix_object["type"],
-                                                                                  name,
-                                                                                  stix_object[name],
-                                                                                  properties[name],
-                                                                                  stix_object["id"] ))
-    return sql_bindings_tuples
-
-
-def generate_insert_for_hashes(hashes, hashes_table_name, **foreign_key):
-    foreign_key_name, foreign_key_value = next(iter(foreign_key.items()))
-
-    bindings = {
-        foreign_key_name: foreign_key_value
-    }
-
-    all_rows_placeholders = []
-    for idx, (hash_name, hash_value) in enumerate(hashes.items()):
-        hash_name_binding_name = "hash_name" + str(idx)
-        hash_value_binding_name = "hash_value" + str(idx)
-
-        all_rows_placeholders.append([
-            f"%({foreign_key_name})s",
-            f"%({hash_name_binding_name})s",
-            f"%({hash_value_binding_name})s"
-        ])
-
-        bindings[hash_name_binding_name] = hash_name
-        bindings[hash_value_binding_name] = hash_value
-
-    all_rows_placeholders_sql = ", ".join(
-        "(" + ", ".join(row_placeholders) + ")"
-        for row_placeholders in all_rows_placeholders
-    )
-
-    sql = f"INSERT INTO {hashes_table_name}" \
-          f"({foreign_key_name}, hash_name, hash_value)" \
-          f" VALUES {all_rows_placeholders_sql}"
-
-    print("HASHES sql (", hashes_table_name, "): ", sql, sep="")
-    print("Hashes:", bindings)
-    return [(sql, bindings)]
-
-
-def generate_insert_for_granular_markings(granular_markings, markings_table_name, **foreign_key):
-    foreign_key_column, foreign_key_value = next(iter(foreign_key.items()))
-
-    bindings = {
-        foreign_key_column: foreign_key_value
-    }
-
-    all_rows_placeholders = []
-    for idx, granular_marking in enumerate(granular_markings):
-        lang_binding_name = f"lang{idx}"
-        marking_ref_binding_name = f"marking_ref{idx}"
-        selectors_binding_name = f"selectors{idx}"
-
-        all_rows_placeholders.append([
-            f"%({foreign_key_column})s",
-            f"%({lang_binding_name})s",
-            f"%({marking_ref_binding_name})s",
-            f"%({selectors_binding_name})s"
-        ])
-
-        bindings[lang_binding_name] = granular_marking.get("lang")
-        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
-        bindings[selectors_binding_name] = granular_marking.get("selectors")
-
-    all_rows_placeholders_sql = ", ".join(
-        "(" + ", ".join(row_placeholders) + ")"
-        for row_placeholders in all_rows_placeholders
-    )
-
-    sql = f'INSERT INTO {markings_table_name}' \
-          f" ({foreign_key_column}, lang, marking_ref, selectors)" \
-          f" VALUES {all_rows_placeholders_sql}"
-
-    print("GRANULAR MARKINGS sql (", markings_table_name, "): ", sql, sep="")
-    print("granular:", bindings)
-    return [(sql, bindings)]
-
-
-def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
-    sql_bindings_tuples = list()
-    for name, ex in extensions.items():
-        sql_bindings_tuples.extend(generate_insert_for_subtype_extension(name,
-                                                                         ex,
-                                                                         foreign_key_value,
-                                                                         type_name,
-                                                                         core_properties))
-    return sql_bindings_tuples
-
-
-def generate_insert_for_subtype_extension(name, ex, foreign_key_value, type_name, core_properties):
-    sql_bindings_tuples = list()
-    properties = ex._properties
-
-    bindings, values = generate_single_values(ex, properties, core_properties)
-    bindings["id"] = foreign_key_value
-    sql = f"INSERT INTO {canonicalize_table_name(type_name, name)}" \
-          f" ({','.join(bindings.keys())})" \
-          f" VALUES ({','.join(values)}, %(id)s)"
-
-    print("sql:", sql)
-    print("ext:", bindings)
-    sql_bindings_tuples.append((sql, bindings))
-    if "external_references" in ex:
-        sql_bindings_tuples.extend(generate_insert_for_external_references(ex["external_references"], ex["id"]))
-
-    for name, prop in properties.items():
-        if table_property(prop, name, core_properties):
-            if name in ex:
-                if embedded_object_list_property(prop, name, core_properties):
-                    sql_bindings_tuples.extend(generate_insert_for_embedded_objects(ex["type"], ex[name], ex["id"]))
-                elif isinstance(prop, DictionaryProperty) and name not in core_properties:
-                    sql_bindings_tuples.extend(generate_insert_for_dictionary_object(type_name, ex[name], name, foreign_key_value))
-                else:
-                    sql_bindings_tuples.extend(generate_insert_for_array_in_table(ex["type"], name, ex[name], properties[name],
-                                                       ex["id"]))
-    return sql_bindings_tuples
-
-
-def generate_insert_for_core(stix_object, core_properties):
-    kind_of_stix_object = "sdo" if "created" in core_properties else "sco"
-    sql_bindings_tuples = list()
-    core_bindings = {}
-
-    for prop_name, value in stix_object.items():
-
-        if prop_name in core_properties:
-            # stored in separate tables, skip here
-            if prop_name not in {
-                "object_marking_refs", "granular_markings", "external_references"
-            }:
-                core_bindings[prop_name] = value
-
-    core_col_names = ", ".join(core_bindings)
-    core_placeholders = ", ".join(
-        f"%({name})s" for name in core_bindings
-    )
-    sql = f"INSERT INTO common.core_{kind_of_stix_object} ({core_col_names}) VALUES ({core_placeholders})"
-    print("CORE sql:", sql)
-    print(core_bindings)
-    sql_bindings_tuples.append((sql, core_bindings))
-
-    if "object_marking_refs" in stix_object:
-        sql_bindings_tuples.extend(generate_insert_for_array_in_table(
-            "core",
-            "object_marking_refs",
-            stix_object.object_marking_refs,
-            stix_object._properties["object_marking_refs"],
-            stix_object.id
-        ))
-
-    # Granular markings
-    if "granular_markings" in stix_object:
-        sql_bindings_tuples.extend(generate_insert_for_granular_markings(
-            stix_object.granular_markings, "common.granular_marking_stix_object",
-            core_id=stix_object.id
-        ))
-
-    if "extensions" in stix_object:
-        sql_bindings_tuples.extend(generate_insert_for_extensions(stix_object.extensions,
-                                                                  stix_object.id,
-                                                                  stix_object.type,
-                                                                  core_properties))
-    return sql_bindings_tuples
-
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index b1551d64..07b7fcfa 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -1,37 +1,22 @@
-from collections import OrderedDict
-
-import inspect
-import re
-
-from sqlalchemy import (
-    ARRAY,
-    TIMESTAMP,
-    Boolean,
-    CheckConstraint,
-    Column,
-    Float,
-    ForeignKey,
-    Integer,
-    LargeBinary,
-    MetaData,
-    Table,
-    Text,
-    create_engine,
-    insert,
+# from collections import OrderedDict
+
+from sqlalchemy import (  # create_engine,; insert,
+    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
+    Integer, LargeBinary, MetaData, Table, Text,
 )
 from sqlalchemy.schema import CreateTable
 
-metadata = MetaData()
-
-
 from stix2.properties import (
-    Property, BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
-    ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty,
-    ListProperty, ObjectReferenceProperty, ReferenceProperty, StringProperty, TimestampProperty, TypeProperty)
+    BinaryProperty, BooleanProperty, DictionaryProperty,
+    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
+    TimestampProperty, TypeProperty,
+)
 from stix2.v21.common import KillChainPhase
 
-from stix2.v21.base import _STIXBase21
-from stix2.v21.base import _DomainObject
+metadata = MetaData()
+
 
 # Helps us know which data goes in core, and which in a type-specific table.
 SCO_COMMON_PROPERTIES = {
@@ -40,9 +25,10 @@
     "spec_version",
     "object_marking_refs",
     "granular_markings",
-    "defanged"
+    "defanged",
 }
 
+
 # Helps us know which data goes in core, and which in a type-specific table.
 SDO_COMMON_PROPERTIES = {
     "id",
@@ -58,7 +44,7 @@
     "labels",
     "confidence",
     "lang",
-    "external_references"
+    "external_references",
 }
 
 
@@ -66,6 +52,7 @@ def canonicalize_table_name_with_schema(schema_name, table_name):
     full_name = schema_name + "." + table_name
     return full_name.replace("-", "_")
 
+
 def canonicalize_table_name(table_name):
     return table_name.replace("-", "_")
 
@@ -80,7 +67,6 @@ def aux_table_property(prop, name, core_properties):
         return False
 
 
-
 def derive_column_name(prop):
     contained_property = prop.contained
     if isinstance(contained_property, ReferenceProperty):
@@ -98,30 +84,36 @@ def get_all_subclasses(cls):
 
     return all_subclasses
 
+
 def create_core_sdo_table():
-    x = Table("core_sdo",
+    x = Table(
+        "core_sdo",
         metadata,
         Column(
             "id",
             Text,
             CheckConstraint(
-                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
             ),
             primary_key=True,
         ),
         Column("spec_version", Text, default="2.1"),
-        Column("created_by_ref",
-               Text,
-               CheckConstraint(
-                   "created_by_ref ~ ^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$"
-                )),
+        Column(
+            "created_by_ref",
+            Text,
+            CheckConstraint(
+                "created_by_ref ~ ^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$",
+                # noqa: E131
+            ),
+        ),
         Column("created", TIMESTAMP(timezone=True)),
         Column("modified", TIMESTAMP(timezone=True)),
         Column("revoked", Boolean),
         Column("confidence", Integer),
         Column("lang", Text),
         Column("object_marking_ref", ARRAY(Text)),
-        schema="common"
+        schema="common",
     )
     print(CreateTable(x))
 
@@ -160,212 +152,299 @@ def generate_table_information(self, name, is_sdo, table_name):
 
 
 @add_method(Property)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     pass
 
 
 @add_method(StringProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  Text,
-                  nullable=not(self.required),
-                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not(self.required),
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
 
 
 @add_method(StringProperty)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     return Text
 
 
 @add_method(IntegerProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  Integer,
-                  nullable=not(self.required),
-                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        Integer,
+        nullable=not(self.required),
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
 
 
 @add_method(IntegerProperty)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     return Integer
 
+
 @add_method(FloatProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  Float,
-                  nullable=not(self.required),
-                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        Float,
+        nullable=not(self.required),
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
 
 @add_method(FloatProperty)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     return Float
 
+
 @add_method(BooleanProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  Boolean,
-                  nullable=not(self.required),
-                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        Boolean,
+        nullable=not(self.required),
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
 
 
 @add_method(BooleanProperty)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     return Boolean
 
 
 @add_method(TypeProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  Text,
-                  nullable=not(self.required),
-                  default=self._fixed_value if hasattr(self, "_fixed_value") else None)
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not(self.required),
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
 
 
 @add_method(IDProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     foreign_key_column = "common.core_sdo.id" if is_sdo else "common.core_sco.id"
-    return Column(name,
-                  Text,
-                  ForeignKey(foreign_key_column, ondelete="CASCADE"),
-                  CheckConstraint(
-                      f"{name} ~ '^{table_name}--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
-                  ),
-                  primary_key=True,
-                  nullable=not (self.required))
+    return Column(
+        name,
+        Text,
+        ForeignKey(foreign_key_column, ondelete="CASCADE"),
+        CheckConstraint(
+            f"{name} ~ '^{table_name}--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+        ),
+        primary_key=True,
+        nullable=not (self.required),
+    )
+
 
 @add_method(EnumProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     enum_re = "|".join(self.allowed)
-    return Column(name,
-                  Text,
-                  CheckConstraint(
-                      f"{name} ~ '^{enum_re}$'"
-                  ),
-                  nullable=not (self.required))
+    return Column(
+        name,
+        Text,
+        CheckConstraint(
+            f"{name} ~ '^{enum_re}$'",
+        ),
+        nullable=not (self.required),
+    )
 
 
 @add_method(TimestampProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  TIMESTAMP(timezone=True),
-                  # CheckConstraint(
-                  #     f"{name} ~ '^{enum_re}$'"
-                  # ),
-                  nullable=not (self.required))
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        TIMESTAMP(timezone=True),
+        # CheckConstraint(
+        #     f"{name} ~ '^{enum_re}$'"
+        # ),
+        nullable=not (self.required),
+    )
+
 
 @add_method(DictionaryProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     columns = list()
-    columns.append(Column("id",
-                          Text,
-                          ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE")))
-    columns.append(Column("name",
-                          Text,
-                          nullable=False))
-    columns.append(Column("value",
-                          Text,
-                          nullable=False))
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE"),
+        ),
+    )
+    columns.append(
+        Column(
+            "name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "value",
+            Text,
+            nullable=False,
+        ),
+    )
     return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
 
+
 @add_method(HashesProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
     columns = list()
-    columns.append(Column("id",
-                          Text,
-                          ForeignKey(canonicalize_table_name(table_name),
-                                     ondelete="CASCADE")))
-    columns.append(Column("hash_name",
-                          Text,
-                          nullable=False))
-    columns.append(Column("hash_value",
-                          Text,
-                          nullable=False))
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(
+                canonicalize_table_name(table_name),
+                ondelete="CASCADE",
+            ),
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_value",
+            Text,
+            nullable=False,
+        ),
+    )
     return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
 
+
 @add_method(HexProperty)
-def generate_table_information(self, name, is_sdo, table_name):
-    return Column(name,
-                  LargeBinary,
-                  nullable=not (self.required))
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+    return Column(
+        name,
+        LargeBinary,
+        nullable=not (self.required),
+    )
+
 
 @add_method(BinaryProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     print("BinaryProperty not handled, yet")
     return None
 
+
 @add_method(ExtensionsProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     columns = list()
-    columns.append(Column("id",
-                          Text,
-                          ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE")))
-    columns.append(Column("ext_table_name",
-                          Text,
-                          nullable=False))
-    columns.append(Column("ext_table_id",
-                          Text,
-                          nullable=False))
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE"),
+        ),
+    )
+    columns.append(
+        Column(
+            "ext_table_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "ext_table_id",
+            Text,
+            nullable=False,
+        ),
+    )
     return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
 
 
 def ref_column(name, specifics):
     if specifics:
         allowed_types = "|".join(specifics)
-        return Column(name,
-                      Text,
-                      CheckConstraint(
-                          f"{name} ~ '^({allowed_types})--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'"
-                      ))
+        return Column(
+            name,
+            Text,
+            CheckConstraint(
+                f"{name} ~ '^({allowed_types})--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+            ),
+        )
     else:
-        return Column(name,
-                      Text,
-                      nullable=False)
+        return Column(
+            name,
+            Text,
+            nullable=False,
+        )
 
 
 @add_method(ReferenceProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     return ref_column(name, self.specifics)
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     return generate_object_table(self.type, table_name)
 
 
 @add_method(ObjectReferenceProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
 @add_method(ListProperty)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     if isinstance(self.contained, ReferenceProperty):
         columns = list()
-        columns.append(Column("id",
-                              Text,
-                              ForeignKey(canonicalize_table_name(table_name),
-                                         ondelete="CASCADE")))
+        columns.append(
+            Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(table_name),
+                    ondelete="CASCADE",
+                ),
+            ),
+        )
         columns.append(ref_column("ref_id", self.contained.specifics))
         return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
     elif isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
-        columns.append(Column("id",
-                              Text,
-                              ForeignKey(canonicalize_table_name(table_name),
-                                         ondelete="CASCADE")))
-        columns.append(Column("ref_id",
-                              Text,
-                              nullable=False))
+        columns.append(
+            Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(table_name),
+                    ondelete="CASCADE",
+                ),
+            ),
+        )
+        columns.append(
+            Column(
+                "ref_id",
+                Text,
+                nullable=False,
+            ),
+        )
         CreateTable(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns))
         return self.contained.generate_table_information(name, False, canonicalize_table_name(table_name + "_" + name))
     else:
         if isinstance(self.contained, Property):
             sql_type = self.contained.determine_sql_type()
             if sql_type:
-                return Column(name,
-                              ARRAY(sql_type),
-                              nullable=not (self.required))
+                return Column(
+                    name,
+                    ARRAY(sql_type),
+                    nullable=not (self.required),
+                )
 
 
 def generate_object_table(stix_object_class, foreign_key_name=None, is_extension=False):
@@ -374,31 +453,33 @@ def generate_object_table(stix_object_class, foreign_key_name=None, is_extension
         table_name = stix_object_class._type
     else:
         table_name = stix_object_class.__name__
-    is_sdo = True # isinstance(stix_object_class, _DomainObject)
+    is_sdo = True  # isinstance(stix_object_class, _DomainObject)
     core_properties = SDO_COMMON_PROPERTIES if is_sdo else SCO_COMMON_PROPERTIES
     columns = list()
     tables = list()
     for name, prop in properties.items():
-        if name == 'id' or name not in core_properties :
-                # or array_property(prop, name, core_properties)):
-                col = prop.generate_table_information(name, is_sdo, table_name)
-                if col is not None and isinstance(col, Column):
-                    columns.append(col)
-                if col is not None and isinstance(col, Table):
-                    tables.append(col)
+        if name == 'id' or name not in core_properties:
+            col = prop.generate_table_information(name, is_sdo, table_name)
+            if col is not None and isinstance(col, Column):
+                columns.append(col)
+            if col is not None and isinstance(col, Table):
+                tables.append(col)
     if is_extension:
         columns.append(Column("id", primary_key=True))
     if foreign_key_name:
-         columns.append(Column("id",
-                               Text,
-                              ForeignKey(canonicalize_table_name(foreign_key_name),
-                                          ondelete="CASCADE")))
-         return Table(canonicalize_table_name(table_name), metadata, *columns)
+        columns.append(
+            Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(foreign_key_name),
+                    ondelete="CASCADE",
+                ),
+            ),
+        )
+        return Table(canonicalize_table_name(table_name), metadata, *columns)
     else:
         x = Table(canonicalize_table_name(table_name), metadata, *columns)
         print(CreateTable(x))
     for t in tables:
         print(CreateTable(t))
-
-
-

From bb6ae81e529c11988df046e2675e5781112df64c Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 19 Mar 2024 09:55:57 -0400
Subject: [PATCH 006/132] changes for Dictionary

---
 .../datastore/relational_db/table_creation.py | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 07b7fcfa..500a4169 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -269,6 +269,7 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 @add_method(DictionaryProperty)
 def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     columns = list()
+
     columns.append(
         Column(
             "id",
@@ -283,13 +284,36 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
             nullable=False,
         ),
     )
-    columns.append(
-        Column(
-            "value",
-            Text,
-            nullable=False,
-        ),
-    )
+    if len(self.specifics) == 1:
+        if self.specifics[0] != "string_list":
+            columns.append(
+                Column(
+                    "value",
+                    Text if self.specifics[0] == "string" else Integer,
+                    nullable=False,
+                ),
+            )
+        else:
+            columns.append(
+                Column(
+                    "value",
+                    ARRAY(Text),
+                    nullable=False,
+                ),
+            )
+    else:
+        columns.append(
+            Column(
+                "string_value",
+                Text
+            ),
+        )
+        columns.append(
+            Column(
+                "integer_value",
+                Integer
+            ),
+        )
     return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
 
 

From 169c4bbff8a6b8686ff3c0314350b8f3741c5a56 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 19 Mar 2024 09:56:59 -0400
Subject: [PATCH 007/132] changes for DictionaryProperty

---
 stix2/properties.py      | 14 +++++++++++++-
 stix2/v21/observables.py | 14 +++++++-------
 2 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index be1c4b69..00f24624 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -387,8 +387,20 @@ def clean(self, value, allow_custom=False):
 
 class DictionaryProperty(Property):
 
-    def __init__(self, spec_version=DEFAULT_VERSION, **kwargs):
+    def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
+
+        if not valid_types:
+            valid_types = ["string"]
+        elif not isinstance(valid_types, list):
+            valid_types = [valid_types]
+
+        for type_ in valid_types:
+            if type_ not in ("string", "integer", "string_list"):
+                raise ValueError("The value of a dictionary key cannot be ", type_)
+
+        self.specifics = valid_types
+
         super(DictionaryProperty, self).__init__(**kwargs)
 
     def clean(self, value, allow_custom=False):
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index f4a4be0f..0929fd32 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -181,7 +181,7 @@ class EmailMessage(_Observable):
         ('message_id', StringProperty()),
         ('subject', StringProperty()),
         ('received_lines', ListProperty(StringProperty)),
-        ('additional_header_fields', DictionaryProperty(spec_version='2.1')),
+        ('additional_header_fields', DictionaryProperty(valid_types="string_list", spec_version='2.1')),
         ('body', StringProperty()),
         ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))),
         ('raw_email_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
@@ -245,7 +245,7 @@ class PDFExt(_Extension):
     _properties = OrderedDict([
         ('version', StringProperty()),
         ('is_optimized', BooleanProperty()),
-        ('document_info_dict', DictionaryProperty(spec_version='2.1')),
+        ('document_info_dict', DictionaryProperty(valid_types="string", spec_version='2.1')),
         ('pdfid0', StringProperty()),
         ('pdfid1', StringProperty()),
     ])
@@ -261,7 +261,7 @@ class RasterImageExt(_Extension):
         ('image_height', IntegerProperty()),
         ('image_width', IntegerProperty()),
         ('bits_per_pixel', IntegerProperty()),
-        ('exif_tags', DictionaryProperty(spec_version='2.1')),
+        ('exif_tags', DictionaryProperty(valid_types=["string", "integer"], spec_version='2.1')),
     ])
 
 
@@ -468,7 +468,7 @@ class HTTPRequestExt(_Extension):
         ('request_method', StringProperty(required=True)),
         ('request_value', StringProperty(required=True)),
         ('request_version', StringProperty()),
-        ('request_header', DictionaryProperty(spec_version='2.1')),
+        ('request_header', DictionaryProperty(valid_types="string_list", spec_version='2.1')),
         ('message_body_length', IntegerProperty()),
         ('message_body_data_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
     ])
@@ -496,7 +496,7 @@ class SocketExt(_Extension):
         ('address_family', EnumProperty(NETWORK_SOCKET_ADDRESS_FAMILY, required=True)),
         ('is_blocking', BooleanProperty()),
         ('is_listening', BooleanProperty()),
-        ('options', DictionaryProperty(spec_version='2.1')),
+        ('options', DictionaryProperty(valid_types="integer", spec_version='2.1')),
         ('socket_type', EnumProperty(NETWORK_SOCKET_TYPE)),
         ('socket_descriptor', IntegerProperty(min=0)),
         ('socket_handle', IntegerProperty()),
@@ -550,7 +550,7 @@ class NetworkTraffic(_Observable):
         ('dst_byte_count', IntegerProperty(min=0)),
         ('src_packets', IntegerProperty(min=0)),
         ('dst_packets', IntegerProperty(min=0)),
-        ('ipfix', DictionaryProperty(spec_version='2.1')),
+        ('ipfix', DictionaryProperty(valid_types=["string", "integer"], spec_version='2.1')),
         ('src_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
         ('dst_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
         ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))),
@@ -634,7 +634,7 @@ class Process(_Observable):
         ('created_time', TimestampProperty()),
         ('cwd', StringProperty()),
         ('command_line', StringProperty()),
-        ('environment_variables', DictionaryProperty(spec_version='2.1')),
+        ('environment_variables', DictionaryProperty(valid_types="string", spec_version='2.1')),
         ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))),
         ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')),
         ('image_ref', ReferenceProperty(valid_types='file', spec_version='2.1')),

From 41f1b7a24355f3f5299982dcd4e3c1899dabf0f0 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 19 Mar 2024 10:12:29 -0400
Subject: [PATCH 008/132] fix existing DictionaryProperty test

---
 stix2/test/v21/test_properties.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 4fb84ec0..7666172f 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -16,7 +16,7 @@
 
 
 def test_dictionary_property():
-    p = DictionaryProperty(StringProperty)
+    p = DictionaryProperty()
 
     assert p.clean({'spec_version': '2.1'})
     with pytest.raises(ValueError):

From f46d523e1734356dd40d6e38c49646e394a60f60 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 19 Mar 2024 10:17:22 -0400
Subject: [PATCH 009/132] fix existing DictionaryProperty test - take 2

---
 stix2/test/v21/test_properties.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 7666172f..b1768e76 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -8,7 +8,7 @@
 from stix2.properties import (
     DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty,
     HashesProperty, IDProperty, ListProperty, ObservableProperty,
-    ReferenceProperty, STIXObjectProperty, StringProperty,
+    ReferenceProperty, STIXObjectProperty,
 )
 from stix2.v21.common import MarkingProperty
 

From dca0070cdfa4af9a35dc72411f8a2d608fa3b1d4 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 20 Mar 2024 12:24:06 -0400
Subject: [PATCH 010/132] add object_kind to custom_extensions

---
 stix2/custom.py     | 4 +++-
 stix2/v21/base.py   | 6 +++++-
 stix2/v21/common.py | 4 ++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/stix2/custom.py b/stix2/custom.py
index 44a03493..9f969b17 100644
--- a/stix2/custom.py
+++ b/stix2/custom.py
@@ -89,7 +89,7 @@ def __init__(self, **kwargs):
     return _CustomObservable
 
 
-def _custom_extension_builder(cls, type, properties, version, base_class):
+def _custom_extension_builder(cls, object_kind, type, properties, version, base_class):
 
     properties = _get_properties_dict(properties)
     toplevel_properties = None
@@ -98,6 +98,7 @@ def _custom_extension_builder(cls, type, properties, version, base_class):
     # it exists.  How to treat the other properties which were given depends on
     # the extension type.
     extension_type = getattr(cls, "extension_type", None)
+    object_kind = object_kind
     if extension_type:
         # I suppose I could also go with a plain string property, since the
         # value is fixed... but an enum property seems more true to the
@@ -128,6 +129,7 @@ class _CustomExtension(cls, base_class):
 
         _type = type
         _properties = nested_properties
+        _object_kind = object_kind
         if extension_type == "toplevel-property-extension":
             _toplevel_properties = toplevel_properties
 
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index 3878b791..002190d5 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -27,7 +27,11 @@ def __init__(self, **kwargs):
 
 
 class _Extension(_Extension, _STIXBase21):
-    extension_type = None
+
+    def __init__(self, object_kind="sco", **kwargs):
+        super(_Extension, self).__init__(**kwargs)
+        self.object_kind = object_kind
+        self.extension_type = None
 
 
 class _DomainObject(_DomainObject, _STIXBase21):
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index 55c4a05d..d1f4bc3d 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -140,11 +140,11 @@ class ExtensionDefinition(_STIXBase21):
     ])
 
 
-def CustomExtension(type='x-custom-ext', properties=None):
+def CustomExtension(object_kind="sco", type='x-custom-ext', properties=None):
     """Custom STIX Object Extension decorator.
     """
     def wrapper(cls):
-        return _custom_extension_builder(cls, type, properties, '2.1', _Extension)
+        return _custom_extension_builder(cls, object_kind, type, properties, '2.1', _Extension)
 
     return wrapper
 

From 3e11107c838bc8743d91e8b4bf4b5da20f8f5a7a Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Wed, 20 Mar 2024 15:28:28 -0400
Subject: [PATCH 011/132] Dictionary Prop Adjustments

---
 stix2/properties.py               | 23 +++++++++++++++++++++++
 stix2/test/v21/test_properties.py | 31 +++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+)

diff --git a/stix2/properties.py b/stix2/properties.py
index 00f24624..26410a70 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -394,6 +394,8 @@ def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
             valid_types = ["string"]
         elif not isinstance(valid_types, list):
             valid_types = [valid_types]
+        elif isinstance(valid_types, list) and 'string_list' in valid_types:
+            raise ValueError("The value of a dictionary key cannot be [\"string_list\"]")
 
         for type_ in valid_types:
             if type_ not in ("string", "integer", "string_list"):
@@ -408,6 +410,8 @@ def clean(self, value, allow_custom=False):
             dictified = _get_dict(value)
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
+        
+        valid_types = self.specifics
         for k in dictified.keys():
             if self.spec_version == '2.0':
                 if len(k) < 3:
@@ -424,6 +428,25 @@ def clean(self, value, allow_custom=False):
                     "underscore (_)"
                 )
                 raise DictionaryKeyError(k, msg)
+            
+            if valid_types == "string":
+                if not isinstance(dictified[k], str):
+                    raise ValueError("The dictionary expects values of type str")
+            elif valid_types == "integer":
+                if not isinstance(dictified[k], int):
+                    raise ValueError("The dictionary expects values of type int")
+            elif valid_types == "string_list":
+                if not isinstance(dictified[k], list):
+                    raise ValueError("The dictionary expects values of type list[str]")
+                for x in dictified[k]:
+                    if not isinstance(x, str):
+                        raise ValueError("The dictionary expects values of type list[str]")
+            else:
+                if not isinstance(dictified[k], list):
+                    raise ValueError("The dictionary expects values of type list[str/int]")
+                for x in dictified[k]:
+                    if not isinstance(x, str) or not isinstance(x, int):
+                        raise ValueError("The dictionary expects values of type list[str/int]")
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index b1768e76..c1b6c373 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -22,6 +22,37 @@ def test_dictionary_property():
     with pytest.raises(ValueError):
         p.clean({})
 
+def test_dictionary_property_values_str():
+    p = DictionaryProperty(valid_types="string", spec_version='2.1', value={'x': '123'})
+    assert p.clean()
+
+    q = DictionaryProperty(valid_types="string", spec_version='2.1', value={'x': 123})
+    with pytest.raises(ValueError):
+        assert q.clean()
+
+def test_dictionary_property_values_int():
+    p = DictionaryProperty(valid_types="integer", spec_version='2.1', value={'x': 123})
+    assert p.clean()
+
+    q = DictionaryProperty(valid_types="integer", spec_version='2.1', value={'x': '123'})
+    with pytest.raises(ValueError):
+        assert q.clean()
+
+def test_dictionary_property_values_stringlist():
+    p = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': ['123', '456']})
+    assert p.clean()
+
+    q = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': '123'})
+    with pytest.raises(ValueError):
+        assert q.clean()
+
+def test_dictionary_property_values_list():
+    p = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': ['123', '456']})
+    assert p.clean()
+
+    q = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': '123'})
+    with pytest.raises(ValueError):
+        assert q.clean()
 
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'

From 0ad3ce796942cc740963300a23a912b6f760abde Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 20 Mar 2024 17:22:30 -0400
Subject: [PATCH 012/132] add args to CustomExtension, implements db class

---
 stix2/custom.py                               |   6 +-
 .../postgres_database_connection.py           |   2 +
 .../datastore/relational_db/relational_db.py  |  70 ++++--
 .../relational_db/relational_db_testing.py    |  26 +-
 .../datastore/relational_db/table_creation.py | 227 +++++++++++-------
 stix2/test/v21/test_properties.py             |   2 +-
 stix2/v20/observables.py                      |   2 +-
 stix2/v21/base.py                             |   8 +-
 stix2/v21/common.py                           |   4 +-
 9 files changed, 210 insertions(+), 137 deletions(-)

diff --git a/stix2/custom.py b/stix2/custom.py
index 9f969b17..cef590d3 100644
--- a/stix2/custom.py
+++ b/stix2/custom.py
@@ -89,7 +89,7 @@ def __init__(self, **kwargs):
     return _CustomObservable
 
 
-def _custom_extension_builder(cls, object_kind, type, properties, version, base_class):
+def _custom_extension_builder(cls, applies_to, type, properties, version, base_class):
 
     properties = _get_properties_dict(properties)
     toplevel_properties = None
@@ -98,7 +98,7 @@ def _custom_extension_builder(cls, object_kind, type, properties, version, base_
     # it exists.  How to treat the other properties which were given depends on
     # the extension type.
     extension_type = getattr(cls, "extension_type", None)
-    object_kind = object_kind
+    applies_to = applies_to
     if extension_type:
         # I suppose I could also go with a plain string property, since the
         # value is fixed... but an enum property seems more true to the
@@ -129,7 +129,7 @@ class _CustomExtension(cls, base_class):
 
         _type = type
         _properties = nested_properties
-        _object_kind = object_kind
+        _applies_to = applies_to
         if extension_type == "toplevel-property-extension":
             _toplevel_properties = toplevel_properties
 
diff --git a/stix2/datastore/relational_db/postgres_database_connection.py b/stix2/datastore/relational_db/postgres_database_connection.py
index 9654c84f..25265897 100644
--- a/stix2/datastore/relational_db/postgres_database_connection.py
+++ b/stix2/datastore/relational_db/postgres_database_connection.py
@@ -1,4 +1,5 @@
 import postgres
+from sqlalchemy import create_engine
 
 from stix2.datastore.relational_db import DatabaseConnection
 
@@ -7,6 +8,7 @@ class PostgresDatabaseConnection(DatabaseConnection):
 
     def __init__(self, host, dbname, user):
         self.db = postgres.Postgres(url=f"host={host} dbname={dbname} user={user}")
+        self.engine = create_engine(f"postgresql://{host}/{dbname}", max_identifier_length=200)
 
     def execute(self, sql_statement, bindings):
         self.db.run(sql_statement, parameters=bindings)
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index aeceb4eb..23156f14 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,9 +1,26 @@
-from stix2.base import _Observable, _STIXBase
+from sqlalchemy import MetaData
+from sqlalchemy.schema import CreateTable
+
+from stix2.base import _STIXBase
 from stix2.datastore import DataSink
-from stix2.datastore.relational_db.sql_bindings_creation import (
-    generate_insert_for_object,
+from stix2.datastore.relational_db.table_creation import (
+    create_core_tables, generate_object_table,
 )
 from stix2.parsing import parse
+from stix2.v21.base import _DomainObject, _Extension, _Observable
+
+
+def _get_all_subclasses(cls):
+    all_subclasses = []
+
+    for subclass in cls.__subclasses__():
+        all_subclasses.append(subclass)
+        all_subclasses.extend(_get_all_subclasses(subclass))
+    return all_subclasses
+
+
+def insert_object(store, stix_obj, is_sdo):
+    pass
 
 
 def _add(store, stix_data, allow_custom=True, version=None):
@@ -40,13 +57,7 @@ def _add(store, stix_data, allow_custom=True, version=None):
         else:
             stix_obj = parse(stix_data, allow_custom, version)
 
-        sql_binding_tuples = generate_insert_for_object(
-            store.database_connection,
-            stix_obj,
-            isinstance(stix_obj, _Observable),
-        )
-        for (sql, bindings) in sql_binding_tuples:
-            store.database_connection.execute(sql, bindings)
+        insert_object(store, stix_obj, isinstance(stix_obj, _Observable))
 
 
 class RelationalDBSink(DataSink):
@@ -73,18 +84,41 @@ class RelationalDBSink(DataSink):
             If part of a MemoryStore, the dict is shared with a MemorySource
 
     """
-    def __init__(self, database_connection, stix_data=None, allow_custom=True, version=None, _store=False):
+    def __init__(
+        self, database_connection, allow_custom=True, version=None,
+        instantiate_database=False,
+    ):
         super(RelationalDBSink, self).__init__()
         self.allow_custom = allow_custom
-
+        self.metadata = MetaData()
         self.database_connection = database_connection
 
-        if _store:
-            self._data = stix_data
-        else:
-            self._data = {}
-            if stix_data:
-                _add(self, stix_data, allow_custom, version)
+        if instantiate_database:
+            self._instantiate_database()
+
+    def _create_table_objects(self):
+        tables = create_core_tables(self.metadata)
+        for sdo_class in _get_all_subclasses(_DomainObject):
+            new_tables = generate_object_table(sdo_class, self.metadata, True)
+            tables.extend(new_tables)
+        for sdo_class in _get_all_subclasses(_Observable):
+            tables.extend(generate_object_table(sdo_class, self.metadata, False))
+        for sdo_class in _get_all_subclasses(_Extension):
+            if hasattr(sdo_class, "_applies_to"):
+                is_sdo = sdo_class._applies_to == "sdo"
+            else:
+                is_sdo = False
+            tables.extend(generate_object_table(sdo_class, self.metadata, is_sdo, is_extension=True))
+        return tables
+
+    def _instantiate_database(self):
+        self._create_table_objects()
+        self.metadata.create_all(self.database_connection.engine)
+
+    def generate_stix_schema(self):
+        tables = self._create_table_objects()
+        for t in tables:
+            print(CreateTable(t).compile(self.database_connection.engine))
 
     def add(self, stix_data, version=None):
         _add(self, stix_data, self.allow_custom, version)
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index d5fc165e..6859070f 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -3,11 +3,10 @@
 import pytz
 
 import stix2
-# from stix2.datastore.relational_db.relational_db import RelationalDBSink
-from stix2.datastore.relational_db.table_creation import (
-    create_core_sdo_table, generate_object_table, get_all_subclasses,
+from stix2.datastore.relational_db.postgres_database_connection import (
+    PostgresDatabaseConnection,
 )
-from stix2.v21.base import _DomainObject, _Extension, _Observable
+from stix2.datastore.relational_db.relational_db import RelationalDBSink
 
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
@@ -98,23 +97,8 @@ def file_example_with_PDFExt_Object():
 
 
 def main():
-    # sink = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
-    # sink.add(directory_stix_object)
-    # sink.add(s)
-    # reg_key = windows_registry_key_example()
-    # sink.add(reg_key)
-    # f = file_example_with_PDFExt_Object()
-    # sink.add(f)
-    # mal = malware_with_all_required_properties()
-    # sink.add(mal)
-
-    create_core_sdo_table()
-    for sdo_class in get_all_subclasses(_DomainObject):
-        generate_object_table(sdo_class)
-    for sdo_class in get_all_subclasses(_Observable):
-        generate_object_table(sdo_class)
-    for sdo_class in get_all_subclasses(_Extension):
-        generate_object_table(sdo_class, is_extension=True)
+    store = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
+    store.generate_stix_schema()
 
 
 if __name__ == '__main__':
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 500a4169..0e4af1ee 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -2,9 +2,8 @@
 
 from sqlalchemy import (  # create_engine,; insert,
     ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, MetaData, Table, Text,
+    Integer, LargeBinary, Table, Text,
 )
-from sqlalchemy.schema import CreateTable
 
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -15,9 +14,6 @@
 )
 from stix2.v21.common import KillChainPhase
 
-metadata = MetaData()
-
-
 # Helps us know which data goes in core, and which in a type-specific table.
 SCO_COMMON_PROPERTIES = {
     "id",
@@ -48,15 +44,14 @@
 }
 
 
-def canonicalize_table_name_with_schema(schema_name, table_name):
-    full_name = schema_name + "." + table_name
+def canonicalize_table_name(table_name, is_sdo):
+    if is_sdo:
+        full_name = ("sdo" if is_sdo else "sco") + "." + table_name
+    else:
+        full_name = table_name
     return full_name.replace("-", "_")
 
 
-def canonicalize_table_name(table_name):
-    return table_name.replace("-", "_")
-
-
 def aux_table_property(prop, name, core_properties):
     if isinstance(prop, ListProperty) and name not in core_properties:
         contained_property = prop.contained
@@ -75,47 +70,76 @@ def derive_column_name(prop):
         return "value"
 
 
-def get_all_subclasses(cls):
-    all_subclasses = []
-
-    for subclass in cls.__subclasses__():
-        all_subclasses.append(subclass)
-        all_subclasses.extend(get_all_subclasses(subclass))
-
-    return all_subclasses
-
-
-def create_core_sdo_table():
-    x = Table(
-        "core_sdo",
+def create_granular_markings_table(metadata, sco_or_sdo):
+    return Table(
+        "common.granular_marking_" + sco_or_sdo,
         metadata,
         Column(
             "id",
             Text,
+            ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        Column("lang", Text),
+        Column(
+            "marking_ref",
+            Text,
             CheckConstraint(
-                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
+                "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
             ),
-            primary_key=True,
         ),
-        Column("spec_version", Text, default="2.1"),
         Column(
-            "created_by_ref",
+            "selectors",
+            ARRAY(Text),
+            CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
+            nullable=False,
+        ),
+        CheckConstraint(
+            """(lang IS NULL AND marking_ref IS NOT NULL)
+               OR
+               (lang IS NOT NULL AND marking_ref IS NULL)""",
+        ),
+    )
+
+
+def create_core_table(metadata, sco_or_sdo):
+    columns = [
+        Column(
+            "id",
             Text,
             CheckConstraint(
-                "created_by_ref ~ ^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$",
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
                 # noqa: E131
             ),
+            primary_key=True,
         ),
-        Column("created", TIMESTAMP(timezone=True)),
-        Column("modified", TIMESTAMP(timezone=True)),
-        Column("revoked", Boolean),
-        Column("confidence", Integer),
-        Column("lang", Text),
+        Column("spec_version", Text, default="2.1"),
         Column("object_marking_ref", ARRAY(Text)),
-        schema="common",
+    ]
+    if sco_or_sdo == "sdo":
+        sdo_columns = [
+            Column(
+                "created_by_ref",
+                Text,
+                CheckConstraint(
+                    "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                    # noqa: E131
+                ),
+            ),
+            Column("created", TIMESTAMP(timezone=True)),
+            Column("modified", TIMESTAMP(timezone=True)),
+            Column("revoked", Boolean),
+            Column("confidence", Integer),
+            Column("lang", Text),
+        ]
+        columns.extend(sdo_columns)
+    else:
+        columns.append(Column("defanged", Boolean, default=False)),
+    return Table(
+        "common.core_" + sco_or_sdo,
+        metadata,
+        *columns
     )
-    print(CreateTable(x))
 
 
 # _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
@@ -147,7 +171,7 @@ def determine_sql_type(self):
 
 
 @add_method(Property)
-def generate_table_information(self, name, is_sdo, table_name):
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):
     pass
 
 
@@ -157,7 +181,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(StringProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         Text,
@@ -172,7 +196,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(IntegerProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         Integer,
@@ -187,7 +211,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(FloatProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         Float,
@@ -202,7 +226,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(BooleanProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         Boolean,
@@ -217,7 +241,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(TypeProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         Text,
@@ -227,7 +251,7 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
 
 @add_method(IDProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     foreign_key_column = "common.core_sdo.id" if is_sdo else "common.core_sco.id"
     return Column(
         name,
@@ -242,7 +266,7 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
 
 @add_method(EnumProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     enum_re = "|".join(self.allowed)
     return Column(
         name,
@@ -255,7 +279,7 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
 
 @add_method(TimestampProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         TIMESTAMP(timezone=True),
@@ -267,14 +291,15 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
 
 @add_method(DictionaryProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     columns = list()
 
     columns.append(
         Column(
             "id",
-            Text,
-            ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE"),
+            Integer if is_extension else Text,
+            ForeignKey(canonicalize_table_name(table_name, is_sdo) + ".id", ondelete="CASCADE"),
+            primary_key=True,
         ),
     )
     columns.append(
@@ -305,30 +330,31 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
         columns.append(
             Column(
                 "string_value",
-                Text
+                Text,
             ),
         )
         columns.append(
             Column(
                 "integer_value",
-                Integer
+                Integer,
             ),
         )
-    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
 
 
 @add_method(HashesProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
 
     columns = list()
     columns.append(
         Column(
             "id",
-            Text,
+            Integer if is_extension else Text,
             ForeignKey(
-                canonicalize_table_name(table_name),
+                canonicalize_table_name(table_name, is_sdo) + ".id",
                 ondelete="CASCADE",
             ),
+            primary_key=True,
         ),
     )
     columns.append(
@@ -345,11 +371,11 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
             nullable=False,
         ),
     )
-    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
 
 
 @add_method(HexProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return Column(
         name,
         LargeBinary,
@@ -358,19 +384,20 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
 
 
 @add_method(BinaryProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     print("BinaryProperty not handled, yet")
     return None
 
 
 @add_method(ExtensionsProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     columns = list()
     columns.append(
         Column(
             "id",
             Text,
-            ForeignKey(canonicalize_table_name(table_name), ondelete="CASCADE"),
+            ForeignKey(canonicalize_table_name(table_name, is_sdo) + ".id", ondelete="CASCADE"),
+            primary_key=True,
         ),
     )
     columns.append(
@@ -383,11 +410,11 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
     columns.append(
         Column(
             "ext_table_id",
-            Text,
+            Integer,
             nullable=False,
         ),
     )
-    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
 
 
 def ref_column(name, specifics):
@@ -409,57 +436,68 @@ def ref_column(name, specifics):
 
 
 @add_method(ReferenceProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     return ref_column(name, self.specifics)
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
-    return generate_object_table(self.type, table_name)
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+    return generate_object_table(self.type, metadata, is_sdo, table_name, is_extension)
 
 
 @add_method(ObjectReferenceProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
 @add_method(ListProperty)
-def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
+def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+    tables = list()
     if isinstance(self.contained, ReferenceProperty):
         columns = list()
         columns.append(
             Column(
                 "id",
-                Text,
+                Integer if is_extension else Text,
                 ForeignKey(
-                    canonicalize_table_name(table_name),
+                    canonicalize_table_name(table_name, is_sdo) + ".id",
                     ondelete="CASCADE",
                 ),
+                primary_key=True,
             ),
         )
         columns.append(ref_column("ref_id", self.contained.specifics))
-        return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns)
+        return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
     elif isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(
             Column(
                 "id",
-                Text,
+                Integer if is_extension else Text,
                 ForeignKey(
-                    canonicalize_table_name(table_name),
+                    canonicalize_table_name(table_name, is_sdo) + ".id",
                     ondelete="CASCADE",
                 ),
+                primary_key=True,
             ),
         )
         columns.append(
             Column(
                 "ref_id",
-                Text,
+                Integer if is_extension else Text,
                 nullable=False,
             ),
         )
-        CreateTable(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns))
-        return self.contained.generate_table_information(name, False, canonicalize_table_name(table_name + "_" + name))
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns))
+        tables.extend(
+            self.contained.generate_table_information(
+                metadata,
+                name,
+                False,
+                canonicalize_table_name(table_name + "_" + name, None),
+            ),
+        )
+        return tables
     else:
         if isinstance(self.contained, Property):
             sql_type = self.contained.determine_sql_type()
@@ -471,39 +509,54 @@ def generate_table_information(self, name, is_sdo, table_name):  # noqa: F811
                 )
 
 
-def generate_object_table(stix_object_class, foreign_key_name=None, is_extension=False):
+def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=None, is_extension=False):
     properties = stix_object_class._properties
     if hasattr(stix_object_class, "_type"):
         table_name = stix_object_class._type
     else:
         table_name = stix_object_class.__name__
-    is_sdo = True  # isinstance(stix_object_class, _DomainObject)
     core_properties = SDO_COMMON_PROPERTIES if is_sdo else SCO_COMMON_PROPERTIES
     columns = list()
     tables = list()
     for name, prop in properties.items():
         if name == 'id' or name not in core_properties:
-            col = prop.generate_table_information(name, is_sdo, table_name)
+            col = prop.generate_table_information(metadata, name, is_sdo, table_name, is_extension=is_extension)
             if col is not None and isinstance(col, Column):
                 columns.append(col)
-            if col is not None and isinstance(col, Table):
-                tables.append(col)
+            if col is not None and isinstance(col, list):
+                tables.extend(col)
     if is_extension:
-        columns.append(Column("id", primary_key=True))
+        columns.append(
+            Column(
+                "id",
+                Integer,
+                # no Foreign Key because it could be for different tables
+                primary_key=True,
+            ),
+        )
     if foreign_key_name:
         columns.append(
             Column(
                 "id",
                 Text,
                 ForeignKey(
-                    canonicalize_table_name(foreign_key_name),
+                    canonicalize_table_name(foreign_key_name, is_sdo) + ".id",
                     ondelete="CASCADE",
                 ),
+                primary_key=True,
             ),
         )
-        return Table(canonicalize_table_name(table_name), metadata, *columns)
+        return [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
     else:
-        x = Table(canonicalize_table_name(table_name), metadata, *columns)
-        print(CreateTable(x))
-    for t in tables:
-        print(CreateTable(t))
+        all_tables = [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
+        all_tables.extend(tables)
+        return all_tables
+
+
+def create_core_tables(metadata):
+    return [
+        create_core_table(metadata, "sdo"),
+        create_granular_markings_table(metadata, "sdo"),
+        create_core_table(metadata, "sco"),
+        create_granular_markings_table(metadata, "sco"),
+    ]
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 4fb84ec0..8432f995 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -16,7 +16,7 @@
 
 
 def test_dictionary_property():
-    p = DictionaryProperty(StringProperty)
+    p = DictionaryProperty(valid_types='string')
 
     assert p.clean({'spec_version': '2.1'})
     with pytest.raises(ValueError):
diff --git a/stix2/v20/observables.py b/stix2/v20/observables.py
index 2b6c81ca..d0bb51c5 100644
--- a/stix2/v20/observables.py
+++ b/stix2/v20/observables.py
@@ -818,5 +818,5 @@ def CustomExtension(type='x-custom-observable-ext', properties=None):
     """Decorator for custom extensions to STIX Cyber Observables.
     """
     def wrapper(cls):
-        return _custom_extension_builder(cls, type, properties, '2.0', _Extension)
+        return _custom_extension_builder(cls, "sco", type, properties, '2.0', _Extension)
     return wrapper
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index 002190d5..c6025a0b 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -27,11 +27,11 @@ def __init__(self, **kwargs):
 
 
 class _Extension(_Extension, _STIXBase21):
-
-    def __init__(self, object_kind="sco", **kwargs):
+    extension_type = None
+    def __init__(self, applies_to="sco", **kwargs):
         super(_Extension, self).__init__(**kwargs)
-        self.object_kind = object_kind
-        self.extension_type = None
+        self._applies_to = applies_to
+
 
 
 class _DomainObject(_DomainObject, _STIXBase21):
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index d1f4bc3d..d18bcc03 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -140,11 +140,11 @@ class ExtensionDefinition(_STIXBase21):
     ])
 
 
-def CustomExtension(object_kind="sco", type='x-custom-ext', properties=None):
+def CustomExtension(type='x-custom-ext', properties=None, applies_to="sco"):
     """Custom STIX Object Extension decorator.
     """
     def wrapper(cls):
-        return _custom_extension_builder(cls, object_kind, type, properties, '2.1', _Extension)
+        return _custom_extension_builder(cls, applies_to, type, properties, '2.1', _Extension)
 
     return wrapper
 

From 57f7ec4ea055ca14a4b2e2f9ecdd2cef1a3a776d Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 21 Mar 2024 17:26:30 -0400
Subject: [PATCH 013/132] fix fk for some extension classes, use kwargs

---
 .../datastore/relational_db/relational_db.py  | 21 ++---
 .../datastore/relational_db/table_creation.py | 77 ++++++++++++-------
 stix2/test/v21/test_properties.py             |  2 +-
 3 files changed, 62 insertions(+), 38 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 23156f14..ae31d495 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -7,7 +7,7 @@
     create_core_tables, generate_object_table,
 )
 from stix2.parsing import parse
-from stix2.v21.base import _DomainObject, _Extension, _Observable
+from stix2.v21.base import _DomainObject, _Extension, _Observable, _RelationshipObject
 
 
 def _get_all_subclasses(cls):
@@ -98,17 +98,20 @@ def __init__(
 
     def _create_table_objects(self):
         tables = create_core_tables(self.metadata)
-        for sdo_class in _get_all_subclasses(_DomainObject):
-            new_tables = generate_object_table(sdo_class, self.metadata, True)
+        for stix_class in _get_all_subclasses(_DomainObject):
+            new_tables = generate_object_table(stix_class, self.metadata, True)
             tables.extend(new_tables)
-        for sdo_class in _get_all_subclasses(_Observable):
-            tables.extend(generate_object_table(sdo_class, self.metadata, False))
-        for sdo_class in _get_all_subclasses(_Extension):
-            if hasattr(sdo_class, "_applies_to"):
-                is_sdo = sdo_class._applies_to == "sdo"
+        for stix_class in _get_all_subclasses(_RelationshipObject):
+            new_tables = generate_object_table(stix_class, self.metadata, True)
+            tables.extend(new_tables)
+        for stix_class in _get_all_subclasses(_Observable):
+            tables.extend(generate_object_table(stix_class, self.metadata, False))
+        for stix_class in _get_all_subclasses(_Extension):
+            if hasattr(stix_class, "_applies_to"):
+                is_sdo = stix_class._applies_to == "sdo"
             else:
                 is_sdo = False
-            tables.extend(generate_object_table(sdo_class, self.metadata, is_sdo, is_extension=True))
+            tables.extend(generate_object_table(stix_class, self.metadata, is_sdo, is_extension=True))
         return tables
 
     def _instantiate_database(self):
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 0e4af1ee..5a7a872b 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -171,7 +171,7 @@ def determine_sql_type(self):
 
 
 @add_method(Property)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):
+def generate_table_information(self, name, **kwargs):
     pass
 
 
@@ -181,7 +181,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(StringProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Text,
@@ -196,7 +196,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(IntegerProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Integer,
@@ -211,7 +211,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(FloatProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Float,
@@ -226,7 +226,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(BooleanProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Boolean,
@@ -241,7 +241,7 @@ def determine_sql_type(self):  # noqa: F811
 
 
 @add_method(TypeProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Text,
@@ -251,8 +251,9 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(IDProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
-    foreign_key_column = "common.core_sdo.id" if is_sdo else "common.core_sco.id"
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    foreign_key_column = "common.core_sdo.id" if kwargs.get("is_sdo") else "common.core_sco.id"
+    table_name = kwargs.get("table_name")
     return Column(
         name,
         Text,
@@ -266,7 +267,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(EnumProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     enum_re = "|".join(self.allowed)
     return Column(
         name,
@@ -279,7 +280,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(TimestampProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         TIMESTAMP(timezone=True),
@@ -291,7 +292,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(DictionaryProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, **kwargs):  # noqa: F811
     columns = list()
 
     columns.append(
@@ -343,7 +344,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(HashesProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, **kwargs):  # noqa: F811
 
     columns = list()
     columns.append(
@@ -375,7 +376,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(HexProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         LargeBinary,
@@ -384,13 +385,13 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
 
 
 @add_method(BinaryProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     print("BinaryProperty not handled, yet")
     return None
 
 
 @add_method(ExtensionsProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, metadata, is_sdo, table_name, **kwargs):  # noqa: F811
     columns = list()
     columns.append(
         Column(
@@ -436,22 +437,24 @@ def ref_column(name, specifics):
 
 
 @add_method(ReferenceProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
     return ref_column(name, self.specifics)
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
-    return generate_object_table(self.type, metadata, is_sdo, table_name, is_extension)
+def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
+    return generate_object_table(self.type, metadata, is_sdo, table_name, is_extension, True, is_list)
 
 
 @add_method(ObjectReferenceProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    table_name = kwargs.get('table_name')
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
 @add_method(ListProperty)
-def generate_table_information(self, metadata, name, is_sdo, table_name, is_extension=False):  # noqa: F811
+def generate_table_information(self, name, metadata, is_sdo, table_name, **kwargs):  # noqa: F811
+    is_extension = kwargs.get('is_extension')
     tables = list()
     if isinstance(self.contained, ReferenceProperty):
         columns = list()
@@ -491,10 +494,12 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
         tables.append(Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns))
         tables.extend(
             self.contained.generate_table_information(
-                metadata,
                 name,
+                metadata,
                 False,
                 canonicalize_table_name(table_name + "_" + name, None),
+                is_extension,
+                is_list=True
             ),
         )
         return tables
@@ -509,7 +514,7 @@ def generate_table_information(self, metadata, name, is_sdo, table_name, is_exte
                 )
 
 
-def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=None, is_extension=False):
+def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=None, is_extension=False, is_embedded_object=False, is_list=False):
     properties = stix_object_class._properties
     if hasattr(stix_object_class, "_type"):
         table_name = stix_object_class._type
@@ -520,12 +525,18 @@ def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=
     tables = list()
     for name, prop in properties.items():
         if name == 'id' or name not in core_properties:
-            col = prop.generate_table_information(metadata, name, is_sdo, table_name, is_extension=is_extension)
+            col = prop.generate_table_information(name,
+                                                  metadata=metadata,
+                                                  is_sdo=is_sdo,
+                                                  table_name=table_name,
+                                                  is_extension=is_extension,
+                                                  is_embedded_object=is_embedded_object,
+                                                  is_list=is_list)
             if col is not None and isinstance(col, Column):
                 columns.append(col)
             if col is not None and isinstance(col, list):
                 tables.extend(col)
-    if is_extension:
+    if (is_extension and not is_embedded_object) or (is_extension and is_embedded_object and is_list):
         columns.append(
             Column(
                 "id",
@@ -535,8 +546,18 @@ def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=
             ),
         )
     if foreign_key_name:
-        columns.append(
-            Column(
+        if is_extension and is_embedded_object and is_list:
+            column = Column(
+                "ref_id",
+                Integer,
+                ForeignKey(
+                    canonicalize_table_name(foreign_key_name, is_sdo) + ".ref_id",
+                    ondelete="CASCADE",
+                ),
+                primary_key=True,
+            )
+        else:
+            column = Column(
                 "id",
                 Text,
                 ForeignKey(
@@ -544,8 +565,8 @@ def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=
                     ondelete="CASCADE",
                 ),
                 primary_key=True,
-            ),
-        )
+            )
+        columns.append(column)
         return [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
     else:
         all_tables = [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 8432f995..9351566a 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -8,7 +8,7 @@
 from stix2.properties import (
     DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty,
     HashesProperty, IDProperty, ListProperty, ObservableProperty,
-    ReferenceProperty, STIXObjectProperty, StringProperty,
+    ReferenceProperty, STIXObjectProperty,
 )
 from stix2.v21.common import MarkingProperty
 

From b89633958b0c8efc1a80153a11a88379c64cfa51 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 25 Mar 2024 19:53:25 -0400
Subject: [PATCH 014/132] remove database connection, start on insert
 statements

---
 stix2/datastore/relational_db/__init__.py     |  35 --
 stix2/datastore/relational_db/add_method.py   |  21 ++
 .../datastore/relational_db/input_creation.py | 270 ++++++++++++++
 .../postgres_database_connection.py           |  17 -
 .../datastore/relational_db/relational_db.py  |  61 ++--
 .../relational_db/relational_db_testing.py    |   5 +-
 .../datastore/relational_db/table_creation.py | 333 +++++++++---------
 stix2/datastore/relational_db/utils.py        |  35 ++
 stix2/v21/base.py                             |   2 +-
 9 files changed, 527 insertions(+), 252 deletions(-)
 delete mode 100644 stix2/datastore/relational_db/__init__.py
 create mode 100644 stix2/datastore/relational_db/add_method.py
 create mode 100644 stix2/datastore/relational_db/input_creation.py
 delete mode 100644 stix2/datastore/relational_db/postgres_database_connection.py
 create mode 100644 stix2/datastore/relational_db/utils.py

diff --git a/stix2/datastore/relational_db/__init__.py b/stix2/datastore/relational_db/__init__.py
deleted file mode 100644
index 1adb8f10..00000000
--- a/stix2/datastore/relational_db/__init__.py
+++ /dev/null
@@ -1,35 +0,0 @@
-from abc import abstractmethod
-
-
-class DatabaseConnection():
-    def __init__(self):
-        pass
-
-    @abstractmethod
-    def execute(self, sql_statement, bindings):
-        """
-
-        Args:
-            sql_statement: the statement to execute
-            bindings: a dictionary where the keys are the column names and the values are the data to be
-            inserted into that column of the table
-
-        Returns:
-
-        """
-        pass
-
-    @abstractmethod
-    def create_insert_statement(self, table_name, bindings, **kwargs):
-        """
-
-        Args:
-            table_name: the name of the table to be inserted into
-            bindings: a dictionary where the keys are the column names and the values are the data to be
-            inserted into that column of the table
-            **kwargs: other specific arguments
-
-        Returns:
-
-        """
-        pass
diff --git a/stix2/datastore/relational_db/add_method.py b/stix2/datastore/relational_db/add_method.py
new file mode 100644
index 00000000..ad08a2e0
--- /dev/null
+++ b/stix2/datastore/relational_db/add_method.py
@@ -0,0 +1,21 @@
+# _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
+#
+#
+# _ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
+
+
+def create_real_method_name(name, klass_name):
+    # if klass_name not in _ALLOWABLE_CLASSES:
+    #     raise NameError
+    # split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
+    # split_up_klass_name.remove("Type")
+    return name + "_" + "_".join([x.lower() for x in klass_name])
+
+
+def add_method(cls):
+    def decorator(fn):
+        method_name = fn.__name__
+        fn.__name__ = create_real_method_name(fn.__name__, cls.__name__)
+        setattr(cls, method_name, fn)
+        return fn
+    return decorator
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
new file mode 100644
index 00000000..ded5cbf5
--- /dev/null
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -0,0 +1,270 @@
+from collections import OrderedDict
+
+from sqlalchemy import (
+    TIMESTAMP,
+    CheckConstraint,
+    Column,
+    ForeignKey,
+    Table,
+    Text,
+    create_engine,
+    insert,
+)
+
+
+from stix2.properties import (
+    DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    IntegerProperty, ListProperty, ReferenceProperty, StringProperty)
+
+from stix2.datastore.relational_db.utils import SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name
+
+
+def single_value(p):
+    return not(isinstance(p, (EmbeddedObjectProperty,
+                              ListProperty,
+                              DictionaryProperty)))
+
+
+def table_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
+    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
+        return True
+    else:
+        return False
+
+
+def embedded_object_list_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return isinstance(contained_property, EmbeddedObjectProperty)
+    else:
+        return False
+
+
+def array_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty, EnumProperty))
+    else:
+        return False
+
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def generate_insert_for_array_in_table(table, property_name, values, prop, foreign_key_value):
+
+    bindings = {
+        "id": foreign_key_value
+    }
+
+    for idx, item in enumerate(values):
+        item_binding_name = f"item{idx}"
+
+        bindings[item_binding_name] = item
+
+    return [insert(table).values(bindings)]
+
+
+def generate_single_values(stix_object, properties, core_properties=[]):
+    bindings = OrderedDict()
+    for name, prop in properties.items():
+        if (single_value(prop) and (name == 'id' or name not in core_properties) or
+                array_property(prop, name, core_properties)):
+            if name in stix_object and name != "type":
+                bindings[name] = stix_object[name] if not array_property(prop, name, core_properties) else "{" + ",".join(
+                    ['"' + x + '"' for x in stix_object[name]]) + "}"
+    return bindings
+
+
+def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
+    bindings = generate_single_values(item, item._properties)
+    bindings["id"] = foreign_key_value
+    sql = f"INSERT INTO {canonicalize_table_name(type_name, item._type)}" \
+          f" ({','.join(bindings.keys())})" \
+          f" VALUES ({','.join(values)}, %(id)s )"
+
+    print("sql:", sql)
+    print("embedded:", bindings)
+    return [(sql, bindings)]
+
+
+def generate_insert_for_dictionary(item, dictionary_table, foreign_key_value, value_types):
+    bindings = {"id": foreign_key_value}
+
+    for idx, (name, value) in enumerate(item.items()):
+        name_binding = f"name{idx}"
+        if len(value_types) == 1:
+            value_binding = f"value{idx}"
+        elif isinstance(value, int):
+            value_binding = f"integer_value{idx}"
+        else:
+            value_binding = f"string_value{idx}"
+
+        bindings[name_binding] = name
+        bindings[value_binding] = value
+
+        return [insert(dictionary_table).values(bindings)]
+
+
+def generate_insert_for_embedded_objects(type_name, values, foreign_key_value):
+    sql_bindings_tuples = list()
+    for item in values:
+        sql_bindings_tuples.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
+    return sql_bindings_tuples
+
+
+def generate_insert_for_hashes(hashes, hashes_table, foreign_key_value):
+    bindings = {"id": foreign_key_value}
+
+    for idx, (hash_name, hash_value) in enumerate(hashes.items()):
+        hash_name_binding_name = "hash_name" + str(idx)
+        hash_value_binding_name = "hash_value" + str(idx)
+
+        bindings[hash_name_binding_name] = hash_name
+        bindings[hash_value_binding_name] = hash_value
+
+    return [insert(hashes_table).values(bindings)]
+
+
+def generate_insert_for_external_references(data_sink, stix_object):
+    insert_statements = list()
+    object_table = data_sink.tables_dictionary["common.external_references"]
+    for er in stix_object["external_references"]:
+        bindings = {"id": stix_object["id"]}
+        for prop in ["source_name", "description", "url", "external_id"]:
+            if prop in er:
+                bindings[prop] = er[prop]
+        er_insert_statement = insert(object_table).values(bindings)
+        insert_statements.append(er_insert_statement)
+
+        if "hashes" in er:
+            hashes_table = data_sink.tables_dictionary[canonicalize_table_name("external_references_hashes", "sdo")]
+            insert_statements.extend(generate_insert_for_hashes(er["hashes"],
+                                                                hashes_table,
+                                                                stix_object["id"]))
+
+    return insert_statements
+
+
+def generate_insert_for_granular_markings(data_sink, stix_object, granular_markings_table):
+    granular_markings = stix_object["granular_markings"]
+    bindings = {
+        "id": stix_object["id"]
+    }
+    for idx, granular_marking in enumerate(granular_markings):
+        lang_binding_name = f"lang{idx}"
+        marking_ref_binding_name = f"marking_ref{idx}"
+        selectors_binding_name = f"selectors{idx}"
+
+        bindings[lang_binding_name] = granular_marking.get("lang")
+        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
+        bindings[selectors_binding_name] = granular_marking.get("selectors")
+
+    return [insert(granular_markings_table).values(bindings)]
+
+
+def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
+    sql_bindings_tuples = list()
+    for name, ex in extensions.items():
+        sql_bindings_tuples.extend(generate_insert_for_subtype_extension(name,
+                                                                         ex,
+                                                                         foreign_key_value,
+                                                                         type_name,
+                                                                         core_properties))
+    return sql_bindings_tuples
+
+
+def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
+    if schema_name == "sdo":
+        core_table = data_sink.tables_dictionary["common.core_sdo"]
+    else:
+        core_table = data_sink.tables_dictionary["common.core_sco"]
+    insert_statements = list()
+    core_bindings = {}
+
+    for prop_name, value in stix_object.items():
+
+        if prop_name in core_properties:
+            # stored in separate tables, skip here
+            if prop_name not in {"object_marking_refs", "granular_markings", "external_references", "type"}:
+                core_bindings[prop_name] = value
+
+    core_insert_statement = insert(core_table).values(core_bindings)
+    insert_statements.append(core_insert_statement)
+
+    if "object_marking_refs" in stix_object:
+        if schema_name == "sdo":
+            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
+        else:
+            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
+        insert_statements.extend(generate_insert_for_array_in_table(data_sink, stix_object, object_markings_ref_table))
+
+    # Granular markings
+    if "granular_markings" in stix_object:
+        if schema_name == "sdo":
+            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
+        else:
+            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
+        granular_input_statements = generate_insert_for_granular_markings(data_sink,
+                                                                          stix_object.granular_markings,
+                                                                          granular_marking_table)
+        insert_statements.extend(granular_input_statements)
+
+
+    return insert_statements
+
+
+def generate_insert_for_object(data_sink, stix_object, schema_name, foreign_key_value=None):
+    insert_statements = list()
+    stix_id = stix_object["id"]
+    if schema_name == "sco":
+        core_properties = SCO_COMMON_PROPERTIES
+    else:
+        core_properties = SDO_COMMON_PROPERTIES
+    type_name = stix_object["type"]
+    table_name = canonicalize_table_name(type_name, schema_name)
+    object_table = data_sink.tables_dictionary[table_name]
+    properties = stix_object._properties
+    insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
+
+    bindings = generate_single_values(stix_object, properties, core_properties)
+    object_insert_statement = insert(object_table).values(bindings)
+    insert_statements.append(object_insert_statement)
+
+    for name, prop in stix_object._properties.items():
+        if isinstance(prop, DictionaryProperty) and not name == "extensions":
+            dictionary_table_name = canonicalize_table_name(type_name + "_" + name, schema_name)
+            dictionary_table = data_sink.tables_dictionary[dictionary_table_name]
+            insert_statements.extend(generate_insert_for_dictionary(stix_object[name], dictionary_table, stix_id))
+
+    if "external_references" in stix_object:
+        insert_statements.extend(generate_insert_for_external_references(data_sink, stix_object, "sdo"))
+
+    if "extensions" in stix_object:
+        for ex in stix_object["extensions"]:
+            insert_statements.extend(generate_insert_for_object(data_sink, ex, schema_name, stix_id))
+    for name, prop in properties.items():
+        if table_property(prop, name, core_properties):
+            if name in stix_object:
+                if embedded_object_list_property(prop, name, core_properties):
+                    insert_statements.extend(generate_insert_for_embedded_objects(name,
+                                                                                  stix_object[name],
+                                                                                  stix_object["id"]))
+                elif isinstance(prop, ExtensionsProperty):
+                    pass
+                else:
+                    insert_statements.extend(generate_insert_for_array_in_table(stix_object["type"],
+                                                                                  name,
+                                                                                  stix_object[name],
+                                                                                  properties[name],
+                                                                                  stix_object["id"] ))
+    return insert_statements
+
diff --git a/stix2/datastore/relational_db/postgres_database_connection.py b/stix2/datastore/relational_db/postgres_database_connection.py
deleted file mode 100644
index 25265897..00000000
--- a/stix2/datastore/relational_db/postgres_database_connection.py
+++ /dev/null
@@ -1,17 +0,0 @@
-import postgres
-from sqlalchemy import create_engine
-
-from stix2.datastore.relational_db import DatabaseConnection
-
-
-class PostgresDatabaseConnection(DatabaseConnection):
-
-    def __init__(self, host, dbname, user):
-        self.db = postgres.Postgres(url=f"host={host} dbname={dbname} user={user}")
-        self.engine = create_engine(f"postgresql://{host}/{dbname}", max_identifier_length=200)
-
-    def execute(self, sql_statement, bindings):
-        self.db.run(sql_statement, parameters=bindings)
-
-    def create_insert_statement(self, table_name, bindings, **kwargs):
-        return f"INSERT INTO {table_name} ({','.join(bindings.keys())}) VALUES ({','.join(kwargs['values'])})"
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index ae31d495..30bc6a04 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,13 +1,13 @@
-from sqlalchemy import MetaData
+from sqlalchemy import MetaData, create_engine
 from sqlalchemy.schema import CreateTable
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink
-from stix2.datastore.relational_db.table_creation import (
-    create_core_tables, generate_object_table,
-)
+from stix2.datastore.relational_db.table_creation import create_core_tables, generate_object_table
+from stix2.datastore.relational_db.input_creation import generate_insert_for_object
+
 from stix2.parsing import parse
-from stix2.v21.base import _DomainObject, _Extension, _Observable, _RelationshipObject
+from stix2.v21.base import (_DomainObject, _Extension, _Observable, _RelationshipObject,)
 
 
 def _get_all_subclasses(cls):
@@ -19,11 +19,7 @@ def _get_all_subclasses(cls):
     return all_subclasses
 
 
-def insert_object(store, stix_obj, is_sdo):
-    pass
-
-
-def _add(store, stix_data, allow_custom=True, version=None):
+def _add(store, stix_data, allow_custom=True, version="2.1"):
     """Add STIX objects to MemoryStore/Sink.
 
     Adds STIX objects to an in-memory dictionary for fast lookup.
@@ -57,7 +53,7 @@ def _add(store, stix_data, allow_custom=True, version=None):
         else:
             stix_obj = parse(stix_data, allow_custom, version)
 
-        insert_object(store, stix_obj, isinstance(stix_obj, _Observable))
+        store.insert_object(stix_obj)
 
 
 class RelationalDBSink(DataSink):
@@ -85,13 +81,18 @@ class RelationalDBSink(DataSink):
 
     """
     def __init__(
-        self, database_connection, allow_custom=True, version=None,
-        instantiate_database=False,
+        self, database_connection_url, allow_custom=True, version=None,
+        instantiate_database=True,
     ):
         super(RelationalDBSink, self).__init__()
         self.allow_custom = allow_custom
         self.metadata = MetaData()
-        self.database_connection = database_connection
+        self.database_connection = create_engine(database_connection_url)
+
+        self.tables = self._create_table_objects()
+        self.tables_dictionary = dict()
+        for t in self.tables:
+            self.tables_dictionary[t.name] = t
 
         if instantiate_database:
             self._instantiate_database()
@@ -99,30 +100,38 @@ def __init__(
     def _create_table_objects(self):
         tables = create_core_tables(self.metadata)
         for stix_class in _get_all_subclasses(_DomainObject):
-            new_tables = generate_object_table(stix_class, self.metadata, True)
+            new_tables = generate_object_table(stix_class, self.metadata, "sdo")
             tables.extend(new_tables)
         for stix_class in _get_all_subclasses(_RelationshipObject):
-            new_tables = generate_object_table(stix_class, self.metadata, True)
+            new_tables = generate_object_table(stix_class, self.metadata, "sro")
             tables.extend(new_tables)
         for stix_class in _get_all_subclasses(_Observable):
-            tables.extend(generate_object_table(stix_class, self.metadata, False))
+            tables.extend(generate_object_table(stix_class, self.metadata, "sco"))
         for stix_class in _get_all_subclasses(_Extension):
-            if hasattr(stix_class, "_applies_to"):
-                is_sdo = stix_class._applies_to == "sdo"
-            else:
-                is_sdo = False
-            tables.extend(generate_object_table(stix_class, self.metadata, is_sdo, is_extension=True))
+            if stix_class.extension_type not in ["new-sdo", "new-sco", "new-sro"]:
+                if hasattr(stix_class, "_applies_to"):
+                    schema_name = stix_class._applies_to
+                else:
+                    schema_name = "sco"
+                tables.extend(generate_object_table(stix_class, self.metadata, schema_name, is_extension=True))
         return tables
 
     def _instantiate_database(self):
-        self._create_table_objects()
         self.metadata.create_all(self.database_connection.engine)
 
     def generate_stix_schema(self):
-        tables = self._create_table_objects()
-        for t in tables:
+        for t in self.tables:
             print(CreateTable(t).compile(self.database_connection.engine))
 
     def add(self, stix_data, version=None):
-        _add(self, stix_data, self.allow_custom, version)
+        _add(self, stix_data)
     add.__doc__ = _add.__doc__
+
+    def insert_object(self, stix_object):
+        schema_name = "sdo" if "created" in stix_object else "sco"
+        with self.database_connection.begin() as trans:
+            statements = generate_insert_for_object(self, stix_object, schema_name)
+            for stmt in statements:
+                print("executing: ", stmt)
+                trans.execute(stmt)
+            trans.commit()
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 6859070f..695f735b 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -3,9 +3,6 @@
 import pytz
 
 import stix2
-from stix2.datastore.relational_db.postgres_database_connection import (
-    PostgresDatabaseConnection,
-)
 from stix2.datastore.relational_db.relational_db import RelationalDBSink
 
 directory_stix_object = stix2.Directory(
@@ -97,7 +94,7 @@ def file_example_with_PDFExt_Object():
 
 
 def main():
-    store = RelationalDBSink(PostgresDatabaseConnection("localhost", "stix-data-sink", "rpiazza"))
+    store = RelationalDBSink("postgresql://localhost/stix-data-sink")
     store.generate_stix_schema()
 
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 5a7a872b..7b935fd1 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -5,6 +5,7 @@
     Integer, LargeBinary, Table, Text,
 )
 
+from stix2.datastore.relational_db.add_method import add_method
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
     EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
@@ -14,43 +15,7 @@
 )
 from stix2.v21.common import KillChainPhase
 
-# Helps us know which data goes in core, and which in a type-specific table.
-SCO_COMMON_PROPERTIES = {
-    "id",
-    # "type",
-    "spec_version",
-    "object_marking_refs",
-    "granular_markings",
-    "defanged",
-}
-
-
-# Helps us know which data goes in core, and which in a type-specific table.
-SDO_COMMON_PROPERTIES = {
-    "id",
-    # "type",
-    "spec_version",
-    "object_marking_refs",
-    "granular_markings",
-    "defanged",
-    "created",
-    "modified",
-    "created_by_ref",
-    "revoked",
-    "labels",
-    "confidence",
-    "lang",
-    "external_references",
-}
-
-
-def canonicalize_table_name(table_name, is_sdo):
-    if is_sdo:
-        full_name = ("sdo" if is_sdo else "sco") + "." + table_name
-    else:
-        full_name = table_name
-    return full_name.replace("-", "_")
-
+from stix2.datastore.relational_db.utils import SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name
 
 def aux_table_property(prop, name, core_properties):
     if isinstance(prop, ListProperty) and name not in core_properties:
@@ -70,6 +35,61 @@ def derive_column_name(prop):
         return "value"
 
 
+def create_object_markings_refs_table(metadata, sco_or_sdo):
+    return create_ref_table(metadata,
+                            {"marking_definition"},
+                            "common.object_marking_refs_" + sco_or_sdo,
+                            "common.core_" + sco_or_sdo + ".id",
+                            0)
+
+
+def create_ref_table(metadata, specifics, table_name, foreign_key_name, auth_type=0):
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(
+                foreign_key_name,
+                ondelete="CASCADE",
+            ),
+            nullable=False,
+        ),
+    )
+    columns.append(ref_column("ref_id", specifics, auth_type))
+    return Table(table_name, metadata, *columns)
+
+
+def create_hashes_table(name, metadata, schema_name, table_name):
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                ondelete="CASCADE",
+            ),
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_value",
+            Text,
+            nullable=False,
+        ),
+    )
+    return Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)
+
+
 def create_granular_markings_table(metadata, sco_or_sdo):
     return Table(
         "common.granular_marking_" + sco_or_sdo,
@@ -102,7 +122,30 @@ def create_granular_markings_table(metadata, sco_or_sdo):
     )
 
 
-def create_core_table(metadata, sco_or_sdo):
+def create_external_references_tables(metadata):
+    columns = [
+        Column(
+            "id",
+            Text,
+            ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
+            CheckConstraint(
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
+            ),
+            primary_key=True
+        ),
+        Column("source_name", Text),
+        Column("description", Text),
+        Column("url", Text),
+        Column("external_id", Text),
+    ]
+    return [
+        Table("common.external_references", metadata, *columns),
+        #  create_hashes_table("hashes", metadata, "common", "external_references")
+    ]
+
+
+def create_core_table(metadata, schema_name):
     columns = [
         Column(
             "id",
@@ -116,7 +159,7 @@ def create_core_table(metadata, sco_or_sdo):
         Column("spec_version", Text, default="2.1"),
         Column("object_marking_ref", ARRAY(Text)),
     ]
-    if sco_or_sdo == "sdo":
+    if schema_name == "sdo":
         sdo_columns = [
             Column(
                 "created_by_ref",
@@ -136,35 +179,12 @@ def create_core_table(metadata, sco_or_sdo):
     else:
         columns.append(Column("defanged", Boolean, default=False)),
     return Table(
-        "common.core_" + sco_or_sdo,
+        "common.core_" + schema_name,
         metadata,
         *columns
     )
 
 
-# _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
-#
-#
-# _ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
-
-
-def create_real_method_name(name, klass_name):
-    # if klass_name not in _ALLOWABLE_CLASSES:
-    #     raise NameError
-    # split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
-    # split_up_klass_name.remove("Type")
-    return name + "_" + "_".join([x.lower() for x in klass_name])
-
-
-def add_method(cls):
-    def decorator(fn):
-        method_name = fn.__name__
-        fn.__name__ = create_real_method_name(fn.__name__, cls.__name__)
-        setattr(cls, method_name, fn)
-        return fn
-    return decorator
-
-
 @add_method(KillChainPhase)
 def determine_sql_type(self):
     return None
@@ -252,14 +272,14 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 @add_method(IDProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    foreign_key_column = "common.core_sdo.id" if kwargs.get("is_sdo") else "common.core_sco.id"
+    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema") else "common.core_sco.id"
     table_name = kwargs.get("table_name")
     return Column(
         name,
         Text,
         ForeignKey(foreign_key_column, ondelete="CASCADE"),
         CheckConstraint(
-            f"{name} ~ '^{table_name}--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
         ),
         primary_key=True,
         nullable=not (self.required),
@@ -292,15 +312,14 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 
 @add_method(DictionaryProperty)
-def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, **kwargs):  # noqa: F811
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     columns = list()
 
     columns.append(
         Column(
             "id",
-            Integer if is_extension else Text,
-            ForeignKey(canonicalize_table_name(table_name, is_sdo) + ".id", ondelete="CASCADE"),
-            primary_key=True,
+            Text,
+            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
         ),
     )
     columns.append(
@@ -340,39 +359,12 @@ def generate_table_information(self, name, metadata, is_sdo, table_name, is_exte
                 Integer,
             ),
         )
-    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
+    return [Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)]
 
 
 @add_method(HashesProperty)
-def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, **kwargs):  # noqa: F811
-
-    columns = list()
-    columns.append(
-        Column(
-            "id",
-            Integer if is_extension else Text,
-            ForeignKey(
-                canonicalize_table_name(table_name, is_sdo) + ".id",
-                ondelete="CASCADE",
-            ),
-            primary_key=True,
-        ),
-    )
-    columns.append(
-        Column(
-            "hash_name",
-            Text,
-            nullable=False,
-        ),
-    )
-    columns.append(
-        Column(
-            "hash_value",
-            Text,
-            nullable=False,
-        ),
-    )
-    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
+    return [create_hashes_table(name, metadata, schema_name, table_name)]
 
 
 @add_method(HexProperty)
@@ -391,14 +383,14 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 
 @add_method(ExtensionsProperty)
-def generate_table_information(self, name, metadata, is_sdo, table_name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     columns = list()
     columns.append(
         Column(
             "id",
             Text,
-            ForeignKey(canonicalize_table_name(table_name, is_sdo) + ".id", ondelete="CASCADE"),
-            primary_key=True,
+            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
+            nullable=False,
         ),
     )
     columns.append(
@@ -408,26 +400,24 @@ def generate_table_information(self, name, metadata, is_sdo, table_name, **kwarg
             nullable=False,
         ),
     )
-    columns.append(
-        Column(
-            "ext_table_id",
-            Integer,
-            nullable=False,
-        ),
-    )
-    return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
+    return [Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)]
 
 
-def ref_column(name, specifics):
+def ref_column(name, specifics, auth_type=0):
     if specifics:
-        allowed_types = "|".join(specifics)
-        return Column(
-            name,
-            Text,
-            CheckConstraint(
-                f"{name} ~ '^({allowed_types})--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-            ),
-        )
+        types = "|".join(specifics)
+        if auth_type == 0:
+            constraint = \
+                CheckConstraint(
+                    f"{name} ~ '^({types})" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+                )
+        else:
+            constraint = \
+                CheckConstraint(
+                    f"(NOT({name} ~ '^({types})') AND ({name} ~" + "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
+                    # noqa: E131
+                )
+        return Column(name, Text, constraint)
     else:
         return Column(
             name,
@@ -438,12 +428,12 @@ def ref_column(name, specifics):
 
 @add_method(ReferenceProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return ref_column(name, self.specifics)
+    return ref_column(name, self.specifics, self.auth_type)
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, metadata, is_sdo, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
-    return generate_object_table(self.type, metadata, is_sdo, table_name, is_extension, True, is_list)
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
+    return generate_object_table(self.type, metadata, schema_name, table_name, is_extension, True, is_list)
 
 
 @add_method(ObjectReferenceProperty)
@@ -452,54 +442,51 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
+def sub_objects(prop_class):
+    for name, prop in prop_class.type._properties.items():
+        if isinstance(prop, (HashesProperty, EmbeddedObjectProperty)):
+            return True
+    return False
+
+
 @add_method(ListProperty)
-def generate_table_information(self, name, metadata, is_sdo, table_name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     is_extension = kwargs.get('is_extension')
     tables = list()
     if isinstance(self.contained, ReferenceProperty):
-        columns = list()
-        columns.append(
-            Column(
-                "id",
-                Integer if is_extension else Text,
-                ForeignKey(
-                    canonicalize_table_name(table_name, is_sdo) + ".id",
-                    ondelete="CASCADE",
-                ),
-                primary_key=True,
-            ),
-        )
-        columns.append(ref_column("ref_id", self.contained.specifics))
-        return [Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns)]
+        return [create_ref_table(metadata,
+                                 self.contained.specifics,
+                                 canonicalize_table_name(table_name + "_" + name, schema_name),
+                                 canonicalize_table_name(table_name, schema_name) + ".id", )]
     elif isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(
             Column(
                 "id",
-                Integer if is_extension else Text,
+                Text,
                 ForeignKey(
-                    canonicalize_table_name(table_name, is_sdo) + ".id",
+                    canonicalize_table_name(table_name, schema_name) + ".id",
                     ondelete="CASCADE",
                 ),
-                primary_key=True,
             ),
         )
         columns.append(
             Column(
                 "ref_id",
-                Integer if is_extension else Text,
+                Integer,
+                primary_key=True,
                 nullable=False,
             ),
         )
-        tables.append(Table(canonicalize_table_name(table_name + "_" + name, is_sdo), metadata, *columns))
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns))
         tables.extend(
             self.contained.generate_table_information(
                 name,
                 metadata,
-                False,
-                canonicalize_table_name(table_name + "_" + name, None),
+                schema_name,
+                canonicalize_table_name(table_name + "_" + name, None),  # if sub_table_needed else canonicalize_table_name(table_name, None),
                 is_extension,
-                is_list=True
+                is_list=True,
             ),
         )
         return tables
@@ -514,70 +501,78 @@ def generate_table_information(self, name, metadata, is_sdo, table_name, **kwarg
                 )
 
 
-def generate_object_table(stix_object_class, metadata, is_sdo, foreign_key_name=None, is_extension=False, is_embedded_object=False, is_list=False):
+def generate_object_table(
+    stix_object_class, metadata, schema_name, foreign_key_name=None,
+    is_extension=False, is_embedded_object=False, is_list=False,
+):
     properties = stix_object_class._properties
     if hasattr(stix_object_class, "_type"):
         table_name = stix_object_class._type
     else:
         table_name = stix_object_class.__name__
-    core_properties = SDO_COMMON_PROPERTIES if is_sdo else SCO_COMMON_PROPERTIES
+    if table_name.startswith("extension-definition"):
+        table_name = table_name[0:30]
+    core_properties = SDO_COMMON_PROPERTIES if schema_name else SCO_COMMON_PROPERTIES
     columns = list()
     tables = list()
     for name, prop in properties.items():
         if name == 'id' or name not in core_properties:
-            col = prop.generate_table_information(name,
-                                                  metadata=metadata,
-                                                  is_sdo=is_sdo,
-                                                  table_name=table_name,
-                                                  is_extension=is_extension,
-                                                  is_embedded_object=is_embedded_object,
-                                                  is_list=is_list)
+            col = prop.generate_table_information(
+                name,
+                metadata=metadata,
+                schema_name=schema_name,
+                table_name=table_name,
+                is_extension=is_extension,
+                is_embedded_object=is_embedded_object,
+                is_list=is_list,
+            )
             if col is not None and isinstance(col, Column):
                 columns.append(col)
             if col is not None and isinstance(col, list):
                 tables.extend(col)
-    if (is_extension and not is_embedded_object) or (is_extension and is_embedded_object and is_list):
+    if (is_extension and not is_embedded_object):  # or (is_extension and is_embedded_object and is_list):
         columns.append(
             Column(
                 "id",
-                Integer,
+                Text,
                 # no Foreign Key because it could be for different tables
                 primary_key=True,
             ),
         )
     if foreign_key_name:
-        if is_extension and is_embedded_object and is_list:
+        if is_extension or (is_embedded_object and is_list):
             column = Column(
-                "ref_id",
-                Integer,
+                "id",
+                Integer if (is_embedded_object and is_list) else Text,
                 ForeignKey(
-                    canonicalize_table_name(foreign_key_name, is_sdo) + ".ref_id",
+                    canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if (is_embedded_object and is_list) else ".id"),
                     ondelete="CASCADE",
                 ),
-                primary_key=True,
             )
         else:
             column = Column(
                 "id",
                 Text,
                 ForeignKey(
-                    canonicalize_table_name(foreign_key_name, is_sdo) + ".id",
+                    canonicalize_table_name(foreign_key_name, schema_name) + ".id",
                     ondelete="CASCADE",
                 ),
-                primary_key=True,
             )
         columns.append(column)
-        return [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
-    else:
-        all_tables = [Table(canonicalize_table_name(table_name, is_sdo), metadata, *columns)]
-        all_tables.extend(tables)
-        return all_tables
+
+    all_tables = [Table(canonicalize_table_name(table_name, schema_name), metadata, *columns)]
+    all_tables.extend(tables)
+    return all_tables
 
 
 def create_core_tables(metadata):
-    return [
+    tables = [
         create_core_table(metadata, "sdo"),
         create_granular_markings_table(metadata, "sdo"),
         create_core_table(metadata, "sco"),
         create_granular_markings_table(metadata, "sco"),
+        create_object_markings_refs_table(metadata, "sdo"),
+        create_object_markings_refs_table(metadata, "sco")
     ]
+    tables.extend(create_external_references_tables(metadata))
+    return tables
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
new file mode 100644
index 00000000..ffa90eaa
--- /dev/null
+++ b/stix2/datastore/relational_db/utils.py
@@ -0,0 +1,35 @@
+# Helps us know which data goes in core, and which in a type-specific table.
+SCO_COMMON_PROPERTIES = {
+    "id",
+    "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged"
+}
+
+# Helps us know which data goes in core, and which in a type-specific table.
+SDO_COMMON_PROPERTIES = {
+    "id",
+    "type",
+    "spec_version",
+    "object_marking_refs",
+    "granular_markings",
+    "defanged",
+    "created",
+    "modified",
+    "created_by_ref",
+    "revoked",
+    "labels",
+    "confidence",
+    "lang",
+    "external_references"
+}
+
+
+def canonicalize_table_name(table_name, schema_name):
+    if schema_name:
+        full_name = schema_name + "." + table_name
+    else:
+        full_name = table_name
+    return full_name.replace("-", "_")
\ No newline at end of file
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index c6025a0b..5f14f1bb 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -28,12 +28,12 @@ def __init__(self, **kwargs):
 
 class _Extension(_Extension, _STIXBase21):
     extension_type = None
+
     def __init__(self, applies_to="sco", **kwargs):
         super(_Extension, self).__init__(**kwargs)
         self._applies_to = applies_to
 
 
-
 class _DomainObject(_DomainObject, _STIXBase21):
     pass
 

From 80c87340923fdfc111d7dd9ba38b0895368e6838 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 26 Mar 2024 09:06:11 -0400
Subject: [PATCH 015/132] flaky

---
 .pre-commit-config.yaml                       |   8 +-
 .../datastore/relational_db/input_creation.py | 129 ++++++++++--------
 .../datastore/relational_db/relational_db.py  |  24 +++-
 .../datastore/relational_db/table_creation.py |  75 +++++-----
 stix2/datastore/relational_db/utils.py        |   8 +-
 stix2/environment.py                          |  16 +--
 stix2/equivalence/graph/__init__.py           |   8 +-
 stix2/equivalence/object/__init__.py          |   6 +-
 stix2/properties.py                           |   4 +-
 stix2/test/test_workbench.py                  |  14 +-
 stix2/test/v20/test_environment.py            |   2 +-
 stix2/test/v20/test_granular_markings.py      |  82 +++++------
 stix2/test/v20/test_object_markings.py        |  50 +++----
 stix2/test/v20/test_utils.py                  |   2 +-
 stix2/test/v20/test_versioning.py             |   2 +-
 stix2/test/v21/test_campaign.py               |   2 +-
 stix2/test/v21/test_environment.py            |   6 +-
 stix2/test/v21/test_granular_markings.py      |  98 ++++++-------
 stix2/test/v21/test_object_markings.py        |  50 +++----
 stix2/test/v21/test_utils.py                  |   2 +-
 stix2/test/v21/test_versioning.py             |   2 +-
 stix2/v20/sro.py                              |   2 +-
 stix2/v21/sro.py                              |   2 +-
 23 files changed, 317 insertions(+), 277 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c39aaf6d..35dff666 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,22 +1,22 @@
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.5.0
     hooks:
     -   id: trailing-whitespace
     -   id: check-merge-conflict
 -   repo: https://github.com/asottile/add-trailing-comma
-    rev: v2.0.2
+    rev: v3.1.0
     hooks:
     -   id: add-trailing-comma
 -   repo: https://github.com/PyCQA/flake8
-    rev: 3.8.4
+    rev: 7.0.0
     hooks:
     -   id: flake8
         name: Check project styling
         args:
         - --max-line-length=160
 -   repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
     hooks:
     -   id: isort
         name: Sort python imports (shows diff)
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index ded5cbf5..22a83fcf 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -1,28 +1,25 @@
 from collections import OrderedDict
 
-from sqlalchemy import (
-    TIMESTAMP,
-    CheckConstraint,
-    Column,
-    ForeignKey,
-    Table,
-    Text,
-    create_engine,
-    insert,
-)
-
+from sqlalchemy import insert
 
+from stix2.datastore.relational_db.utils import (
+    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+)
 from stix2.properties import (
-    DictionaryProperty, EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
-    IntegerProperty, ListProperty, ReferenceProperty, StringProperty)
-
-from stix2.datastore.relational_db.utils import SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name
+    DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
+    ExtensionsProperty, FloatProperty, IntegerProperty, ListProperty,
+    ReferenceProperty, StringProperty,
+)
 
 
 def single_value(p):
-    return not(isinstance(p, (EmbeddedObjectProperty,
-                              ListProperty,
-                              DictionaryProperty)))
+    return not isinstance(
+        p, (
+            EmbeddedObjectProperty,
+            ListProperty,
+            DictionaryProperty,
+        ),
+    )
 
 
 def table_property(prop, name, core_properties):
@@ -59,10 +56,10 @@ def derive_column_name(prop):
         return "value"
 
 
-def generate_insert_for_array_in_table(table, property_name, values, prop, foreign_key_value):
+def generate_insert_for_array_in_table(table, values, foreign_key_value):
 
     bindings = {
-        "id": foreign_key_value
+        "id": foreign_key_value,
     }
 
     for idx, item in enumerate(values):
@@ -76,24 +73,20 @@ def generate_insert_for_array_in_table(table, property_name, values, prop, forei
 def generate_single_values(stix_object, properties, core_properties=[]):
     bindings = OrderedDict()
     for name, prop in properties.items():
-        if (single_value(prop) and (name == 'id' or name not in core_properties) or
-                array_property(prop, name, core_properties)):
+        if (
+            single_value(prop) and (name == 'id' or name not in core_properties) or
+            array_property(prop, name, core_properties)
+        ):
             if name in stix_object and name != "type":
                 bindings[name] = stix_object[name] if not array_property(prop, name, core_properties) else "{" + ",".join(
-                    ['"' + x + '"' for x in stix_object[name]]) + "}"
+                    ['"' + x + '"' for x in stix_object[name]],
+                ) + "}"
     return bindings
 
 
 def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
     bindings = generate_single_values(item, item._properties)
     bindings["id"] = foreign_key_value
-    sql = f"INSERT INTO {canonicalize_table_name(type_name, item._type)}" \
-          f" ({','.join(bindings.keys())})" \
-          f" VALUES ({','.join(values)}, %(id)s )"
-
-    print("sql:", sql)
-    print("embedded:", bindings)
-    return [(sql, bindings)]
 
 
 def generate_insert_for_dictionary(item, dictionary_table, foreign_key_value, value_types):
@@ -147,17 +140,21 @@ def generate_insert_for_external_references(data_sink, stix_object):
 
         if "hashes" in er:
             hashes_table = data_sink.tables_dictionary[canonicalize_table_name("external_references_hashes", "sdo")]
-            insert_statements.extend(generate_insert_for_hashes(er["hashes"],
-                                                                hashes_table,
-                                                                stix_object["id"]))
+            insert_statements.extend(
+                generate_insert_for_hashes(
+                    er["hashes"],
+                    hashes_table,
+                    stix_object["id"],
+                ),
+            )
 
     return insert_statements
 
 
-def generate_insert_for_granular_markings(data_sink, stix_object, granular_markings_table):
+def generate_insert_for_granular_markings(granular_markings_table, stix_object):
     granular_markings = stix_object["granular_markings"]
     bindings = {
-        "id": stix_object["id"]
+        "id": stix_object["id"],
     }
     for idx, granular_marking in enumerate(granular_markings):
         lang_binding_name = f"lang{idx}"
@@ -171,15 +168,19 @@ def generate_insert_for_granular_markings(data_sink, stix_object, granular_marki
     return [insert(granular_markings_table).values(bindings)]
 
 
-def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
-    sql_bindings_tuples = list()
-    for name, ex in extensions.items():
-        sql_bindings_tuples.extend(generate_insert_for_subtype_extension(name,
-                                                                         ex,
-                                                                         foreign_key_value,
-                                                                         type_name,
-                                                                         core_properties))
-    return sql_bindings_tuples
+# def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
+#     sql_bindings_tuples = list()
+#     for name, ex in extensions.items():
+#         sql_bindings_tuples.extend(
+#             generate_insert_for_subtype_extension(
+#                 name,
+#                 ex,
+#                 foreign_key_value,
+#                 type_name,
+#                 core_properties,
+#             ),
+#         )
+#     return sql_bindings_tuples
 
 
 def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
@@ -205,7 +206,13 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
             object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
         else:
             object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
-        insert_statements.extend(generate_insert_for_array_in_table(data_sink, stix_object, object_markings_ref_table))
+        insert_statements.extend(
+            generate_insert_for_array_in_table(
+                object_markings_ref_table,
+                stix_object["object_marking_refs"],
+                stix_object["id"],
+            ),
+        )
 
     # Granular markings
     if "granular_markings" in stix_object:
@@ -213,12 +220,12 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
             granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
         else:
             granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
-        granular_input_statements = generate_insert_for_granular_markings(data_sink,
-                                                                          stix_object.granular_markings,
-                                                                          granular_marking_table)
+        granular_input_statements = generate_insert_for_granular_markings(
+            granular_marking_table,
+            stix_object.granular_markings,
+        )
         insert_statements.extend(granular_input_statements)
 
-
     return insert_statements
 
 
@@ -255,16 +262,22 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, foreign_key_
         if table_property(prop, name, core_properties):
             if name in stix_object:
                 if embedded_object_list_property(prop, name, core_properties):
-                    insert_statements.extend(generate_insert_for_embedded_objects(name,
-                                                                                  stix_object[name],
-                                                                                  stix_object["id"]))
+                    insert_statements.extend(
+                        generate_insert_for_embedded_objects(
+                            name,
+                            stix_object[name],
+                            stix_object["id"],
+                        ),
+                    )
                 elif isinstance(prop, ExtensionsProperty):
                     pass
                 else:
-                    insert_statements.extend(generate_insert_for_array_in_table(stix_object["type"],
-                                                                                  name,
-                                                                                  stix_object[name],
-                                                                                  properties[name],
-                                                                                  stix_object["id"] ))
+                    insert_statements.extend(
+                        generate_insert_for_array_in_table(
+                            stix_object["type"],
+                            name,
+                            stix_object[name],
+                            properties[name],
+                            stix_object["id"], ),
+                    )
     return insert_statements
-
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 30bc6a04..d05fb08e 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,13 +1,18 @@
 from sqlalchemy import MetaData, create_engine
-from sqlalchemy.schema import CreateTable
+from sqlalchemy.schema import CreateSchema, CreateTable
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink
-from stix2.datastore.relational_db.table_creation import create_core_tables, generate_object_table
-from stix2.datastore.relational_db.input_creation import generate_insert_for_object
-
+from stix2.datastore.relational_db.input_creation import (
+    generate_insert_for_object,
+)
+from stix2.datastore.relational_db.table_creation import (
+    create_core_tables, generate_object_table,
+)
 from stix2.parsing import parse
-from stix2.v21.base import (_DomainObject, _Extension, _Observable, _RelationshipObject,)
+from stix2.v21.base import (
+    _DomainObject, _Extension, _Observable, _RelationshipObject,
+)
 
 
 def _get_all_subclasses(cls):
@@ -89,6 +94,8 @@ def __init__(
         self.metadata = MetaData()
         self.database_connection = create_engine(database_connection_url)
 
+        self._create_schemas()
+
         self.tables = self._create_table_objects()
         self.tables_dictionary = dict()
         for t in self.tables:
@@ -97,6 +104,13 @@ def __init__(
         if instantiate_database:
             self._instantiate_database()
 
+    def _create_schemas(self):
+        with self.database_connection.begin() as trans:
+            trans.execute(CreateSchema("common", if_not_exists=True))
+            trans.execute(CreateSchema("sdo", if_not_exists=True))
+            trans.execute(CreateSchema("sco", if_not_exists=True))
+            trans.execute(CreateSchema("sro", if_not_exists=True))
+
     def _create_table_objects(self):
         tables = create_core_tables(self.metadata)
         for stix_class in _get_all_subclasses(_DomainObject):
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 7b935fd1..0f343c3f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -6,6 +6,9 @@
 )
 
 from stix2.datastore.relational_db.add_method import add_method
+from stix2.datastore.relational_db.utils import (
+    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+)
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
     EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
@@ -15,7 +18,6 @@
 )
 from stix2.v21.common import KillChainPhase
 
-from stix2.datastore.relational_db.utils import SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name
 
 def aux_table_property(prop, name, core_properties):
     if isinstance(prop, ListProperty) and name not in core_properties:
@@ -36,14 +38,17 @@ def derive_column_name(prop):
 
 
 def create_object_markings_refs_table(metadata, sco_or_sdo):
-    return create_ref_table(metadata,
-                            {"marking_definition"},
-                            "common.object_marking_refs_" + sco_or_sdo,
-                            "common.core_" + sco_or_sdo + ".id",
-                            0)
+    return create_ref_table(
+        metadata,
+        {"marking_definition"},
+        "object_marking_refs_" + sco_or_sdo,
+        "common.core_" + sco_or_sdo + ".id",
+        "common",
+        0,
+    )
 
 
-def create_ref_table(metadata, specifics, table_name, foreign_key_name, auth_type=0):
+def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_name, auth_type=0):
     columns = list()
     columns.append(
         Column(
@@ -57,7 +62,7 @@ def create_ref_table(metadata, specifics, table_name, foreign_key_name, auth_typ
         ),
     )
     columns.append(ref_column("ref_id", specifics, auth_type))
-    return Table(table_name, metadata, *columns)
+    return Table(table_name, metadata, *columns, schema=schema_name)
 
 
 def create_hashes_table(name, metadata, schema_name, table_name):
@@ -87,12 +92,12 @@ def create_hashes_table(name, metadata, schema_name, table_name):
             nullable=False,
         ),
     )
-    return Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
 
 
 def create_granular_markings_table(metadata, sco_or_sdo):
     return Table(
-        "common.granular_marking_" + sco_or_sdo,
+        "granular_marking_" + sco_or_sdo,
         metadata,
         Column(
             "id",
@@ -119,6 +124,7 @@ def create_granular_markings_table(metadata, sco_or_sdo):
                OR
                (lang IS NOT NULL AND marking_ref IS NULL)""",
         ),
+        schema="common",
     )
 
 
@@ -132,7 +138,7 @@ def create_external_references_tables(metadata):
                 "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
                 # noqa: E131
             ),
-            primary_key=True
+            primary_key=True,
         ),
         Column("source_name", Text),
         Column("description", Text),
@@ -140,7 +146,7 @@ def create_external_references_tables(metadata):
         Column("external_id", Text),
     ]
     return [
-        Table("common.external_references", metadata, *columns),
+        Table("external_references", metadata, *columns, schema="common"),
         #  create_hashes_table("hashes", metadata, "common", "external_references")
     ]
 
@@ -179,9 +185,10 @@ def create_core_table(metadata, schema_name):
     else:
         columns.append(Column("defanged", Boolean, default=False)),
     return Table(
-        "common.core_" + schema_name,
+        "core_" + schema_name,
         metadata,
-        *columns
+        *columns,
+        schema="common",
     )
 
 
@@ -205,7 +212,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Text,
-        nullable=not(self.required),
+        nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
@@ -220,7 +227,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Integer,
-        nullable=not(self.required),
+        nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
@@ -235,7 +242,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Float,
-        nullable=not(self.required),
+        nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
@@ -250,7 +257,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Boolean,
-        nullable=not(self.required),
+        nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
@@ -265,7 +272,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
         Text,
-        nullable=not(self.required),
+        nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
@@ -359,7 +366,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
                 Integer,
             ),
         )
-    return [Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)]
+    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
 
 
 @add_method(HashesProperty)
@@ -400,7 +407,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
             nullable=False,
         ),
     )
-    return [Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns)]
+    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
 
 
 def ref_column(name, specifics, auth_type=0):
@@ -409,13 +416,14 @@ def ref_column(name, specifics, auth_type=0):
         if auth_type == 0:
             constraint = \
                 CheckConstraint(
-                    f"{name} ~ '^({types})" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+                    f"{name} ~ '^({types})" +
+                    "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
                 )
         else:
             constraint = \
                 CheckConstraint(
-                    f"(NOT({name} ~ '^({types})') AND ({name} ~" + "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
-                    # noqa: E131
+                    f"(NOT({name} ~ '^({types})') AND ({name} ~ " +
+                    "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
                 )
         return Column(name, Text, constraint)
     else:
@@ -454,10 +462,15 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
     is_extension = kwargs.get('is_extension')
     tables = list()
     if isinstance(self.contained, ReferenceProperty):
-        return [create_ref_table(metadata,
-                                 self.contained.specifics,
-                                 canonicalize_table_name(table_name + "_" + name, schema_name),
-                                 canonicalize_table_name(table_name, schema_name) + ".id", )]
+        return [
+            create_ref_table(
+                metadata,
+                self.contained.specifics,
+                canonicalize_table_name(table_name + "_" + name),
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                "common",
+            ),
+        ]
     elif isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(
@@ -478,7 +491,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 nullable=False,
             ),
         )
-        tables.append(Table(canonicalize_table_name(table_name + "_" + name, schema_name), metadata, *columns))
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
         tables.extend(
             self.contained.generate_table_information(
                 name,
@@ -560,7 +573,7 @@ def generate_object_table(
             )
         columns.append(column)
 
-    all_tables = [Table(canonicalize_table_name(table_name, schema_name), metadata, *columns)]
+    all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
     all_tables.extend(tables)
     return all_tables
 
@@ -572,7 +585,7 @@ def create_core_tables(metadata):
         create_core_table(metadata, "sco"),
         create_granular_markings_table(metadata, "sco"),
         create_object_markings_refs_table(metadata, "sdo"),
-        create_object_markings_refs_table(metadata, "sco")
+        create_object_markings_refs_table(metadata, "sco"),
     ]
     tables.extend(create_external_references_tables(metadata))
     return tables
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index ffa90eaa..e908ce89 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -5,7 +5,7 @@
     "spec_version",
     "object_marking_refs",
     "granular_markings",
-    "defanged"
+    "defanged",
 }
 
 # Helps us know which data goes in core, and which in a type-specific table.
@@ -23,13 +23,13 @@
     "labels",
     "confidence",
     "lang",
-    "external_references"
+    "external_references",
 }
 
 
-def canonicalize_table_name(table_name, schema_name):
+def canonicalize_table_name(table_name, schema_name=None):
     if schema_name:
         full_name = schema_name + "." + table_name
     else:
         full_name = table_name
-    return full_name.replace("-", "_")
\ No newline at end of file
+    return full_name.replace("-", "_")
diff --git a/stix2/environment.py b/stix2/environment.py
index eab2ba9e..76d7be0d 100644
--- a/stix2/environment.py
+++ b/stix2/environment.py
@@ -191,7 +191,7 @@ def creator_of(self, obj):
     def object_similarity(
         obj1, obj2, prop_scores={}, ds1=None, ds2=None,
         ignore_spec_version=False, versioning_checks=False,
-        max_depth=1, **weight_dict
+        max_depth=1, **weight_dict,
     ):
         """This method returns a measure of how similar the two objects are.
 
@@ -236,14 +236,14 @@ def object_similarity(
         """
         return object_similarity(
             obj1, obj2, prop_scores, ds1, ds2, ignore_spec_version,
-            versioning_checks, max_depth, **weight_dict
+            versioning_checks, max_depth, **weight_dict,
         )
 
     @staticmethod
     def object_equivalence(
         obj1, obj2, prop_scores={}, threshold=70, ds1=None, ds2=None,
         ignore_spec_version=False, versioning_checks=False,
-        max_depth=1, **weight_dict
+        max_depth=1, **weight_dict,
     ):
         """This method returns a true/false value if two objects are semantically equivalent.
         Internally, it calls the object_similarity function and compares it against the given
@@ -294,13 +294,13 @@ def object_equivalence(
         """
         return object_equivalence(
             obj1, obj2, prop_scores, threshold, ds1, ds2,
-            ignore_spec_version, versioning_checks, max_depth, **weight_dict
+            ignore_spec_version, versioning_checks, max_depth, **weight_dict,
         )
 
     @staticmethod
     def graph_similarity(
         ds1, ds2, prop_scores={}, ignore_spec_version=False,
-        versioning_checks=False, max_depth=1, **weight_dict
+        versioning_checks=False, max_depth=1, **weight_dict,
     ):
         """This method returns a similarity score for two given graphs.
         Each DataStore can contain a connected or disconnected graph and the
@@ -347,14 +347,14 @@ def graph_similarity(
         """
         return graph_similarity(
             ds1, ds2, prop_scores, ignore_spec_version,
-            versioning_checks, max_depth, **weight_dict
+            versioning_checks, max_depth, **weight_dict,
         )
 
     @staticmethod
     def graph_equivalence(
         ds1, ds2, prop_scores={}, threshold=70,
         ignore_spec_version=False, versioning_checks=False,
-        max_depth=1, **weight_dict
+        max_depth=1, **weight_dict,
     ):
         """This method returns a true/false value if two graphs are semantically equivalent.
         Internally, it calls the graph_similarity function and compares it against the given
@@ -403,5 +403,5 @@ def graph_equivalence(
         """
         return graph_equivalence(
             ds1, ds2, prop_scores, threshold, ignore_spec_version,
-            versioning_checks, max_depth, **weight_dict
+            versioning_checks, max_depth, **weight_dict,
         )
diff --git a/stix2/equivalence/graph/__init__.py b/stix2/equivalence/graph/__init__.py
index 1f46fd3e..1beee4aa 100644
--- a/stix2/equivalence/graph/__init__.py
+++ b/stix2/equivalence/graph/__init__.py
@@ -11,7 +11,7 @@
 def graph_equivalence(
     ds1, ds2, prop_scores={}, threshold=70,
     ignore_spec_version=False, versioning_checks=False,
-    max_depth=1, **weight_dict
+    max_depth=1, **weight_dict,
 ):
     """This method returns a true/false value if two graphs are semantically equivalent.
     Internally, it calls the graph_similarity function and compares it against the given
@@ -60,7 +60,7 @@ def graph_equivalence(
     """
     similarity_result = graph_similarity(
         ds1, ds2, prop_scores, ignore_spec_version,
-        versioning_checks, max_depth, **weight_dict
+        versioning_checks, max_depth, **weight_dict,
     )
     if similarity_result >= threshold:
         return True
@@ -69,7 +69,7 @@ def graph_equivalence(
 
 def graph_similarity(
     ds1, ds2, prop_scores={}, ignore_spec_version=False,
-    versioning_checks=False, max_depth=1, **weight_dict
+    versioning_checks=False, max_depth=1, **weight_dict,
 ):
     """This method returns a similarity score for two given graphs.
     Each DataStore can contain a connected or disconnected graph and the
@@ -147,7 +147,7 @@ def graph_similarity(
         result = object_similarity(
             object1, object2, iprop_score, ds1, ds2,
             ignore_spec_version, versioning_checks,
-            max_depth, **weights
+            max_depth, **weights,
         )
 
         if object1_id not in results:
diff --git a/stix2/equivalence/object/__init__.py b/stix2/equivalence/object/__init__.py
index 2b16a346..00534881 100644
--- a/stix2/equivalence/object/__init__.py
+++ b/stix2/equivalence/object/__init__.py
@@ -14,7 +14,7 @@
 def object_equivalence(
     obj1, obj2, prop_scores={}, threshold=70, ds1=None,
     ds2=None, ignore_spec_version=False,
-    versioning_checks=False, max_depth=1, **weight_dict
+    versioning_checks=False, max_depth=1, **weight_dict,
 ):
     """This method returns a true/false value if two objects are semantically equivalent.
     Internally, it calls the object_similarity function and compares it against the given
@@ -65,7 +65,7 @@ def object_equivalence(
     """
     similarity_result = object_similarity(
         obj1, obj2, prop_scores, ds1, ds2, ignore_spec_version,
-        versioning_checks, max_depth, **weight_dict
+        versioning_checks, max_depth, **weight_dict,
     )
     if similarity_result >= threshold:
         return True
@@ -75,7 +75,7 @@ def object_equivalence(
 def object_similarity(
     obj1, obj2, prop_scores={}, ds1=None, ds2=None,
     ignore_spec_version=False, versioning_checks=False,
-    max_depth=1, **weight_dict
+    max_depth=1, **weight_dict,
 ):
     """This method returns a measure of similarity depending on how
     similar the two objects are.
diff --git a/stix2/properties.py b/stix2/properties.py
index 00f24624..24a83eb6 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -588,13 +588,13 @@ def clean(self, value, allow_custom):
 
         if auth_type == self._WHITELIST:
             type_ok = is_stix_type(
-                obj_type, self.spec_version, *generics
+                obj_type, self.spec_version, *generics,
             ) or obj_type in specifics
 
         else:
             type_ok = (
                 not is_stix_type(
-                    obj_type, self.spec_version, *generics
+                    obj_type, self.spec_version, *generics,
                 ) and obj_type not in specifics
             ) or obj_type in blacklist_exceptions
 
diff --git a/stix2/test/test_workbench.py b/stix2/test/test_workbench.py
index 84f97a59..dc3f66f5 100644
--- a/stix2/test/test_workbench.py
+++ b/stix2/test/test_workbench.py
@@ -32,7 +32,7 @@ def test_workbench_environment():
 
     # Create a STIX object
     ind = create(
-        Indicator, id=constants.INDICATOR_ID, **constants.INDICATOR_KWARGS
+        Indicator, id=constants.INDICATOR_ID, **constants.INDICATOR_KWARGS,
     )
     save(ind)
 
@@ -50,7 +50,7 @@ def test_workbench_environment():
 
 def test_workbench_get_all_attack_patterns():
     mal = AttackPattern(
-        id=constants.ATTACK_PATTERN_ID, **constants.ATTACK_PATTERN_KWARGS
+        id=constants.ATTACK_PATTERN_ID, **constants.ATTACK_PATTERN_KWARGS,
     )
     save(mal)
 
@@ -70,7 +70,7 @@ def test_workbench_get_all_campaigns():
 
 def test_workbench_get_all_courses_of_action():
     coa = CourseOfAction(
-        id=constants.COURSE_OF_ACTION_ID, **constants.COURSE_OF_ACTION_KWARGS
+        id=constants.COURSE_OF_ACTION_ID, **constants.COURSE_OF_ACTION_KWARGS,
     )
     save(coa)
 
@@ -114,7 +114,7 @@ def test_workbench_get_all_infrastructures():
 
 def test_workbench_get_all_intrusion_sets():
     ins = IntrusionSet(
-        id=constants.INTRUSION_SET_ID, **constants.INTRUSION_SET_KWARGS
+        id=constants.INTRUSION_SET_ID, **constants.INTRUSION_SET_KWARGS,
     )
     save(ins)
 
@@ -161,7 +161,7 @@ def test_workbench_get_all_notes():
 
 def test_workbench_get_all_observed_data():
     od = ObservedData(
-        id=constants.OBSERVED_DATA_ID, **constants.OBSERVED_DATA_KWARGS
+        id=constants.OBSERVED_DATA_ID, **constants.OBSERVED_DATA_KWARGS,
     )
     save(od)
 
@@ -190,7 +190,7 @@ def test_workbench_get_all_reports():
 
 def test_workbench_get_all_threat_actors():
     thr = ThreatActor(
-        id=constants.THREAT_ACTOR_ID, **constants.THREAT_ACTOR_KWARGS
+        id=constants.THREAT_ACTOR_ID, **constants.THREAT_ACTOR_KWARGS,
     )
     save(thr)
 
@@ -210,7 +210,7 @@ def test_workbench_get_all_tools():
 
 def test_workbench_get_all_vulnerabilities():
     vuln = Vulnerability(
-        id=constants.VULNERABILITY_ID, **constants.VULNERABILITY_KWARGS
+        id=constants.VULNERABILITY_ID, **constants.VULNERABILITY_KWARGS,
     )
     save(vuln)
 
diff --git a/stix2/test/v20/test_environment.py b/stix2/test/v20/test_environment.py
index b6f3a0b3..8dd4eca7 100644
--- a/stix2/test/v20/test_environment.py
+++ b/stix2/test/v20/test_environment.py
@@ -459,7 +459,7 @@ def test_semantic_check_with_versioning(ds, ds2):
                 },
             ],
             object_marking_refs=[stix2.v20.TLP_WHITE],
-        )
+        ),
     )
     ds.add(ind)
     score = stix2.equivalence.object.reference_check(ind.id, INDICATOR_ID, ds, ds2, **weights)
diff --git a/stix2/test/v20/test_granular_markings.py b/stix2/test/v20/test_granular_markings.py
index ae2da3b0..e20d5685 100644
--- a/stix2/test/v20/test_granular_markings.py
+++ b/stix2/test/v20/test_granular_markings.py
@@ -15,7 +15,7 @@
 
 def test_add_marking_mark_one_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -28,7 +28,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"])
 
@@ -47,7 +47,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": MARKING_IDS[0],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -60,7 +60,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": MARKING_IDS[0],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -73,7 +73,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": TLP_RED.id,
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             TLP_RED,
         ),
@@ -91,7 +91,7 @@ def test_add_marking_mark_multiple_selector_one_refs(data):
 
 def test_add_marking_mark_multiple_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -104,7 +104,7 @@ def test_add_marking_mark_multiple_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "name"])
 
@@ -120,7 +120,7 @@ def test_add_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -129,7 +129,7 @@ def test_add_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0]], ["name"])
 
@@ -145,7 +145,7 @@ def test_add_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -154,7 +154,7 @@ def test_add_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0]], ["description"])
 
@@ -391,7 +391,7 @@ def test_get_markings_positional_arguments_combinations(data):
                         "marking_ref": MARKING_IDS[1],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[1]],
         ),
@@ -407,7 +407,7 @@ def test_get_markings_positional_arguments_combinations(data):
                         "marking_ref": MARKING_IDS[1],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[1]],
         ),
@@ -426,7 +426,7 @@ def test_remove_marking_remove_multiple_selector_one_ref():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, MARKING_IDS[0], ["description", "modified"])
     assert "granular_markings" not in before
@@ -440,7 +440,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -449,7 +449,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -468,7 +468,7 @@ def test_remove_marking_mark_one_selector_markings_from_multiple_ones():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -481,7 +481,7 @@ def test_remove_marking_mark_one_selector_markings_from_multiple_ones():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -500,7 +500,7 @@ def test_remove_marking_mark_mutilple_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"])
     assert "granular_markings" not in before
@@ -514,7 +514,7 @@ def test_remove_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -527,7 +527,7 @@ def test_remove_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -542,7 +542,7 @@ def test_remove_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["description"])
     assert "granular_markings" not in before
@@ -572,7 +572,7 @@ def test_remove_marking_not_present():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(MarkingNotFoundError):
         markings.remove_markings(before, [MARKING_IDS[1]], ["description"])
@@ -594,7 +594,7 @@ def test_remove_marking_not_present():
                 "marking_ref": MARKING_IDS[3],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
     dict(
         granular_markings=[
@@ -611,7 +611,7 @@ def test_remove_marking_not_present():
                 "marking_ref": MARKING_IDS[3],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
 ]
 
@@ -844,14 +844,14 @@ def test_create_sdo_with_invalid_marking():
                     "marking_ref": MARKING_IDS[0],
                 },
             ],
-            **MALWARE_KWARGS
+            **MALWARE_KWARGS,
         )
     assert str(excinfo.value) == "Selector foo in Malware is not valid!"
 
 
 def test_set_marking_mark_one_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -864,7 +864,7 @@ def test_set_marking_mark_one_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"])
     for m in before["granular_markings"]:
@@ -879,7 +879,7 @@ def test_set_marking_mark_multiple_selector_one_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -888,7 +888,7 @@ def test_set_marking_mark_multiple_selector_one_refs():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0]], ["description", "modified"])
     for m in before["granular_markings"]:
@@ -897,7 +897,7 @@ def test_set_marking_mark_multiple_selector_one_refs():
 
 def test_set_marking_mark_multiple_selector_multiple_refs_from_none():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -910,7 +910,7 @@ def test_set_marking_mark_multiple_selector_multiple_refs_from_none():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"])
     for m in before["granular_markings"]:
@@ -925,7 +925,7 @@ def test_set_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -938,7 +938,7 @@ def test_set_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[1], MARKING_IDS[2]], ["description"])
 
@@ -962,7 +962,7 @@ def test_set_marking_bad_selector(marking):
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -971,7 +971,7 @@ def test_set_marking_bad_selector(marking):
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     with pytest.raises(InvalidSelectorError):
@@ -988,7 +988,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -997,7 +997,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0]], ["description"])
     for m in before["granular_markings"]:
@@ -1020,7 +1020,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
     dict(
         granular_markings=[
@@ -1037,7 +1037,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
 ]
 
@@ -1099,7 +1099,7 @@ def test_set_marking_on_id_property():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     assert "id" in malware["granular_markings"][0]["selectors"]
diff --git a/stix2/test/v20/test_object_markings.py b/stix2/test/v20/test_object_markings.py
index 6bd2269d..88b3f14b 100644
--- a/stix2/test/v20/test_object_markings.py
+++ b/stix2/test/v20/test_object_markings.py
@@ -26,7 +26,7 @@
             Malware(**MALWARE_KWARGS),
             Malware(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -34,7 +34,7 @@
             MALWARE_KWARGS,
             dict(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -42,7 +42,7 @@
             Malware(**MALWARE_KWARGS),
             Malware(
                 object_marking_refs=[TLP_AMBER.id],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             TLP_AMBER,
         ),
@@ -60,12 +60,12 @@ def test_add_markings_one_marking(data):
 
 def test_add_markings_multiple_marking():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     after = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], None)
@@ -76,7 +76,7 @@ def test_add_markings_multiple_marking():
 
 def test_add_markings_combination():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]],
@@ -90,7 +90,7 @@ def test_add_markings_combination():
                 "marking_ref": MARKING_IDS[3],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.add_markings(before, MARKING_IDS[0], None)
@@ -114,7 +114,7 @@ def test_add_markings_combination():
 )
 def test_add_markings_bad_markings(data):
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(exceptions.InvalidValueError):
         before = markings.add_markings(before, data, None)
@@ -274,14 +274,14 @@ def test_get_markings_object_and_granular_combinations(data):
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -306,33 +306,33 @@ def test_remove_markings_object_level(data):
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[2]],
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             dict(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[2]],
         ),
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], TLP_AMBER.id],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], TLP_AMBER],
         ),
@@ -350,7 +350,7 @@ def test_remove_markings_multiple(data):
 def test_remove_markings_bad_markings():
     before = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(MarkingNotFoundError) as excinfo:
         markings.remove_markings(before, [MARKING_IDS[4]], None)
@@ -362,14 +362,14 @@ def test_remove_markings_bad_markings():
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -533,14 +533,14 @@ def test_is_marked_object_and_granular_combinations():
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -557,11 +557,11 @@ def test_is_marked_no_markings(data):
 def test_set_marking():
     before = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[4], MARKING_IDS[5]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.set_markings(before, [MARKING_IDS[4], MARKING_IDS[5]], None)
@@ -585,11 +585,11 @@ def test_set_marking():
 def test_set_marking_bad_input(data):
     before = Malware(
         object_marking_refs=[MARKING_IDS[0]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[0]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(exceptions.InvalidValueError):
         before = markings.set_markings(before, data, None)
diff --git a/stix2/test/v20/test_utils.py b/stix2/test/v20/test_utils.py
index f61369b9..e3b15279 100644
--- a/stix2/test/v20/test_utils.py
+++ b/stix2/test/v20/test_utils.py
@@ -201,7 +201,7 @@ def test_deduplicate(stix_objs1):
 def test_find_property_index(object, tuple_to_find, expected_index):
     assert stix2.serialization.find_property_index(
         object,
-        *tuple_to_find
+        *tuple_to_find,
     ) == expected_index
 
 
diff --git a/stix2/test/v20/test_versioning.py b/stix2/test/v20/test_versioning.py
index d3973f03..ace1cc82 100644
--- a/stix2/test/v20/test_versioning.py
+++ b/stix2/test/v20/test_versioning.py
@@ -43,7 +43,7 @@ def test_making_new_version_with_embedded_object():
             "source_name": "capec",
             "external_id": "CAPEC-163",
         }],
-        **CAMPAIGN_MORE_KWARGS
+        **CAMPAIGN_MORE_KWARGS,
     )
 
     campaign_v2 = campaign_v1.new_version(
diff --git a/stix2/test/v21/test_campaign.py b/stix2/test/v21/test_campaign.py
index edc7d777..5fc8e4bb 100644
--- a/stix2/test/v21/test_campaign.py
+++ b/stix2/test/v21/test_campaign.py
@@ -21,7 +21,7 @@
 
 def test_campaign_example():
     campaign = stix2.v21.Campaign(
-        **CAMPAIGN_MORE_KWARGS
+        **CAMPAIGN_MORE_KWARGS,
     )
 
     assert campaign.serialize(pretty=True) == EXPECTED
diff --git a/stix2/test/v21/test_environment.py b/stix2/test/v21/test_environment.py
index 51ca15a5..502f2b77 100644
--- a/stix2/test/v21/test_environment.py
+++ b/stix2/test/v21/test_environment.py
@@ -973,7 +973,7 @@ def test_semantic_check_with_versioning(ds, ds2):
                 },
             ],
             object_marking_refs=[stix2.v21.TLP_WHITE],
-        )
+        ),
     )
     ds.add(ind)
     score = stix2.equivalence.object.reference_check(ind.id, INDICATOR_ID, ds, ds2, **weights)
@@ -1146,7 +1146,7 @@ def test_depth_limiting():
     }
     prop_scores1 = {}
     env1 = stix2.equivalence.graph.graph_similarity(
-        mem_store1, mem_store2, prop_scores1, **custom_weights
+        mem_store1, mem_store2, prop_scores1, **custom_weights,
     )
 
     assert round(env1) == 38
@@ -1159,7 +1159,7 @@ def test_depth_limiting():
     # Switching parameters
     prop_scores2 = {}
     env2 = stix2.equivalence.graph.graph_similarity(
-        mem_store2, mem_store1, prop_scores2, **custom_weights
+        mem_store2, mem_store1, prop_scores2, **custom_weights,
     )
 
     assert round(env2) == 38
diff --git a/stix2/test/v21/test_granular_markings.py b/stix2/test/v21/test_granular_markings.py
index ff8fe26d..2f5bae8c 100644
--- a/stix2/test/v21/test_granular_markings.py
+++ b/stix2/test/v21/test_granular_markings.py
@@ -14,7 +14,7 @@
 
 def test_add_marking_mark_one_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -27,7 +27,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"])
 
@@ -46,7 +46,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": MARKING_IDS[0],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -59,7 +59,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": MARKING_IDS[0],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -72,7 +72,7 @@ def test_add_marking_mark_one_selector_multiple_refs():
                         "marking_ref": TLP_RED.id,
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             TLP_RED,
         ),
@@ -90,7 +90,7 @@ def test_add_marking_mark_multiple_selector_one_refs(data):
 
 def test_add_marking_mark_multiple_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -103,7 +103,7 @@ def test_add_marking_mark_multiple_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "name"])
 
@@ -113,7 +113,7 @@ def test_add_marking_mark_multiple_selector_multiple_refs():
 
 def test_add_marking_mark_multiple_selector_multiple_refs_mixed():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -134,7 +134,7 @@ def test_add_marking_mark_multiple_selector_multiple_refs_mixed():
                 "lang": MARKING_LANGS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1], MARKING_LANGS[0], MARKING_LANGS[1]], ["description", "name"])
 
@@ -150,7 +150,7 @@ def test_add_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -159,7 +159,7 @@ def test_add_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0]], ["name"])
 
@@ -175,7 +175,7 @@ def test_add_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -184,7 +184,7 @@ def test_add_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.add_markings(before, [MARKING_IDS[0]], ["description"])
 
@@ -513,7 +513,7 @@ def test_get_markings_multiple_selectors_with_options(data):
                         "marking_ref": MARKING_IDS[1],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[1]],
         ),
@@ -529,7 +529,7 @@ def test_get_markings_multiple_selectors_with_options(data):
                         "marking_ref": MARKING_IDS[1],
                     },
                 ],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[1]],
         ),
@@ -548,7 +548,7 @@ def test_remove_marking_remove_multiple_selector_one_ref():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, MARKING_IDS[0], ["description", "modified"])
     assert "granular_markings" not in before
@@ -562,7 +562,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -571,7 +571,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -590,7 +590,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones_mixed():
                 "lang": MARKING_LANGS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -603,7 +603,7 @@ def test_remove_marking_mark_one_selector_from_multiple_ones_mixed():
                 "lang": MARKING_LANGS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_LANGS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -622,7 +622,7 @@ def test_remove_marking_mark_one_selector_markings_from_multiple_ones():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -635,7 +635,7 @@ def test_remove_marking_mark_one_selector_markings_from_multiple_ones():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -654,7 +654,7 @@ def test_remove_marking_mark_mutilple_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"])
     assert "granular_markings" not in before
@@ -668,7 +668,7 @@ def test_remove_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = Malware(
         granular_markings=[
@@ -681,7 +681,7 @@ def test_remove_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["modified"])
     for m in before["granular_markings"]:
@@ -696,7 +696,7 @@ def test_remove_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.remove_markings(before, [MARKING_IDS[0]], ["description"])
     assert "granular_markings" not in before
@@ -726,7 +726,7 @@ def test_remove_marking_not_present():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(MarkingNotFoundError):
         markings.remove_markings(before, [MARKING_IDS[1]], ["description"])
@@ -752,7 +752,7 @@ def test_remove_marking_not_present():
                 "lang": MARKING_LANGS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
     dict(
         granular_markings=[
@@ -773,7 +773,7 @@ def test_remove_marking_not_present():
                 "lang": MARKING_LANGS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
 ]
 
@@ -1008,14 +1008,14 @@ def test_create_sdo_with_invalid_marking():
                     "marking_ref": MARKING_IDS[0],
                 },
             ],
-            **MALWARE_KWARGS
+            **MALWARE_KWARGS,
         )
     assert str(excinfo.value) == "Selector foo in Malware is not valid!"
 
 
 def test_set_marking_mark_one_selector_multiple_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1028,7 +1028,7 @@ def test_set_marking_mark_one_selector_multiple_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description"])
     for m in before["granular_markings"]:
@@ -1037,7 +1037,7 @@ def test_set_marking_mark_one_selector_multiple_refs():
 
 def test_set_marking_mark_one_selector_multiple_lang_refs():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1050,7 +1050,7 @@ def test_set_marking_mark_one_selector_multiple_lang_refs():
                 "lang": MARKING_LANGS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_LANGS[0], MARKING_LANGS[1]], ["description"])
     for m in before["granular_markings"]:
@@ -1065,7 +1065,7 @@ def test_set_marking_mark_multiple_selector_one_refs():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1074,7 +1074,7 @@ def test_set_marking_mark_multiple_selector_one_refs():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0]], ["description", "modified"])
     for m in before["granular_markings"]:
@@ -1093,7 +1093,7 @@ def test_set_marking_mark_multiple_mixed_markings():
                 "lang": MARKING_LANGS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1106,7 +1106,7 @@ def test_set_marking_mark_multiple_mixed_markings():
                 "lang": MARKING_LANGS[3],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[2], MARKING_LANGS[3]], ["description", "modified"])
     for m in before["granular_markings"]:
@@ -1115,7 +1115,7 @@ def test_set_marking_mark_multiple_mixed_markings():
 
 def test_set_marking_mark_multiple_selector_multiple_refs_from_none():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1128,7 +1128,7 @@ def test_set_marking_mark_multiple_selector_multiple_refs_from_none():
                 "marking_ref": MARKING_IDS[1],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], ["description", "modified"])
     for m in before["granular_markings"]:
@@ -1143,7 +1143,7 @@ def test_set_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1156,7 +1156,7 @@ def test_set_marking_mark_another_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[1], MARKING_IDS[2]], ["description"])
 
@@ -1180,7 +1180,7 @@ def test_set_marking_bad_selector(marking):
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1189,7 +1189,7 @@ def test_set_marking_bad_selector(marking):
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     with pytest.raises(InvalidSelectorError):
@@ -1206,7 +1206,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         granular_markings=[
@@ -1215,7 +1215,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     before = markings.set_markings(before, [MARKING_IDS[0]], ["description"])
     for m in before["granular_markings"]:
@@ -1238,7 +1238,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
     dict(
         granular_markings=[
@@ -1255,7 +1255,7 @@ def test_set_marking_mark_same_property_same_marking():
                 "marking_ref": MARKING_IDS[2],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     ),
 ]
 
@@ -1317,7 +1317,7 @@ def test_set_marking_on_id_property():
                 "marking_ref": MARKING_IDS[0],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     assert "id" in malware["granular_markings"][0]["selectors"]
diff --git a/stix2/test/v21/test_object_markings.py b/stix2/test/v21/test_object_markings.py
index bb1c4ab0..63273fb7 100644
--- a/stix2/test/v21/test_object_markings.py
+++ b/stix2/test/v21/test_object_markings.py
@@ -25,7 +25,7 @@
             Malware(**MALWARE_KWARGS),
             Malware(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -33,7 +33,7 @@
             MALWARE_KWARGS,
             dict(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MARKING_IDS[0],
         ),
@@ -41,7 +41,7 @@
             Malware(**MALWARE_KWARGS),
             Malware(
                 object_marking_refs=[TLP_AMBER.id],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             TLP_AMBER,
         ),
@@ -59,12 +59,12 @@ def test_add_markings_one_marking(data):
 
 def test_add_markings_multiple_marking():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     after = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.add_markings(before, [MARKING_IDS[0], MARKING_IDS[1]], None)
@@ -75,7 +75,7 @@ def test_add_markings_multiple_marking():
 
 def test_add_markings_combination():
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1]],
@@ -89,7 +89,7 @@ def test_add_markings_combination():
                 "marking_ref": MARKING_IDS[3],
             },
         ],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.add_markings(before, MARKING_IDS[0], None)
@@ -113,7 +113,7 @@ def test_add_markings_combination():
 )
 def test_add_markings_bad_markings(data):
     before = Malware(
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(exceptions.InvalidValueError):
         before = markings.add_markings(before, data, None)
@@ -273,14 +273,14 @@ def test_get_markings_object_and_granular_combinations(data):
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -305,33 +305,33 @@ def test_remove_markings_object_level(data):
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[2]],
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             dict(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], MARKING_IDS[2]],
         ),
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], TLP_AMBER.id],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(
                 object_marking_refs=[MARKING_IDS[1]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             [MARKING_IDS[0], TLP_AMBER],
         ),
@@ -349,7 +349,7 @@ def test_remove_markings_multiple(data):
 def test_remove_markings_bad_markings():
     before = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(MarkingNotFoundError) as excinfo:
         markings.remove_markings(before, [MARKING_IDS[4]], None)
@@ -361,14 +361,14 @@ def test_remove_markings_bad_markings():
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -532,14 +532,14 @@ def test_is_marked_object_and_granular_combinations():
         (
             Malware(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             Malware(**MALWARE_KWARGS),
         ),
         (
             dict(
                 object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-                **MALWARE_KWARGS
+                **MALWARE_KWARGS,
             ),
             MALWARE_KWARGS,
         ),
@@ -556,11 +556,11 @@ def test_is_marked_no_markings(data):
 def test_set_marking():
     before = Malware(
         object_marking_refs=[MARKING_IDS[0], MARKING_IDS[1], MARKING_IDS[2]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[4], MARKING_IDS[5]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
 
     before = markings.set_markings(before, [MARKING_IDS[4], MARKING_IDS[5]], None)
@@ -584,11 +584,11 @@ def test_set_marking():
 def test_set_marking_bad_input(data):
     before = Malware(
         object_marking_refs=[MARKING_IDS[0]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     after = Malware(
         object_marking_refs=[MARKING_IDS[0]],
-        **MALWARE_KWARGS
+        **MALWARE_KWARGS,
     )
     with pytest.raises(exceptions.InvalidValueError):
         before = markings.set_markings(before, data, None)
diff --git a/stix2/test/v21/test_utils.py b/stix2/test/v21/test_utils.py
index 33e7ea49..6dad23e8 100644
--- a/stix2/test/v21/test_utils.py
+++ b/stix2/test/v21/test_utils.py
@@ -205,7 +205,7 @@ def test_deduplicate(stix_objs1):
 def test_find_property_index(object, tuple_to_find, expected_index):
     assert stix2.serialization.find_property_index(
         object,
-        *tuple_to_find
+        *tuple_to_find,
     ) == expected_index
 
 
diff --git a/stix2/test/v21/test_versioning.py b/stix2/test/v21/test_versioning.py
index c7b6f119..98e01383 100644
--- a/stix2/test/v21/test_versioning.py
+++ b/stix2/test/v21/test_versioning.py
@@ -48,7 +48,7 @@ def test_making_new_version_with_embedded_object():
             "source_name": "capec",
             "external_id": "CAPEC-163",
         }],
-        **CAMPAIGN_MORE_KWARGS
+        **CAMPAIGN_MORE_KWARGS,
     )
 
     campaign_v2 = campaign_v1.new_version(
diff --git a/stix2/v20/sro.py b/stix2/v20/sro.py
index 1372a5e5..13d5e136 100644
--- a/stix2/v20/sro.py
+++ b/stix2/v20/sro.py
@@ -39,7 +39,7 @@ class Relationship(_RelationshipObject):
     # Explicitly define the first three kwargs to make readable Relationship declarations.
     def __init__(
         self, source_ref=None, relationship_type=None,
-        target_ref=None, **kwargs
+        target_ref=None, **kwargs,
     ):
         # Allow (source_ref, relationship_type, target_ref) as positional args.
         if source_ref and not kwargs.get('source_ref'):
diff --git a/stix2/v21/sro.py b/stix2/v21/sro.py
index bf636c3d..8ef1582c 100644
--- a/stix2/v21/sro.py
+++ b/stix2/v21/sro.py
@@ -46,7 +46,7 @@ class Relationship(_RelationshipObject):
     # Explicitly define the first three kwargs to make readable Relationship declarations.
     def __init__(
         self, source_ref=None, relationship_type=None,
-        target_ref=None, **kwargs
+        target_ref=None, **kwargs,
     ):
         # Allow (source_ref, relationship_type, target_ref) as positional args.
         if source_ref and not kwargs.get('source_ref'):

From 5c707b541500d7e92694704de77d6fac58d609a0 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Tue, 26 Mar 2024 09:56:37 -0400
Subject: [PATCH 016/132] More Dict Prop edits

---
 stix2/properties.py               | 29 ++++++++++++-----------------
 stix2/test/v21/test_properties.py | 13 ++++++++++---
 2 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 26410a70..2ddeadd6 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -392,10 +392,11 @@ def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
 
         if not valid_types:
             valid_types = ["string"]
-        elif not isinstance(valid_types, list):
+        elif not isinstance(valid_types, ListProperty):
             valid_types = [valid_types]
-        elif isinstance(valid_types, list) and 'string_list' in valid_types:
-            raise ValueError("The value of a dictionary key cannot be [\"string_list\"]")
+        
+        if 'string_list' in valid_types and len(valid_types) > 1:
+            raise ValueError("'string_list' cannot be combined with other types in a list.")
 
         for type_ in valid_types:
             if type_ not in ("string", "integer", "string_list"):
@@ -429,24 +430,18 @@ def clean(self, value, allow_custom=False):
                 )
                 raise DictionaryKeyError(k, msg)
             
-            if valid_types == "string":
-                if not isinstance(dictified[k], str):
+            if "string" in valid_types:
+                if not isinstance(dictified[k], StringProperty):
                     raise ValueError("The dictionary expects values of type str")
-            elif valid_types == "integer":
-                if not isinstance(dictified[k], int):
+            elif "integer" in valid_types:
+                if not isinstance(dictified[k], IntegerProperty):
                     raise ValueError("The dictionary expects values of type int")
-            elif valid_types == "string_list":
-                if not isinstance(dictified[k], list):
+            elif "string_list" in valid_types:
+                if not isinstance(dictified[k], ListProperty(StringProperty)):
                     raise ValueError("The dictionary expects values of type list[str]")
-                for x in dictified[k]:
-                    if not isinstance(x, str):
-                        raise ValueError("The dictionary expects values of type list[str]")
             else:
-                if not isinstance(dictified[k], list):
-                    raise ValueError("The dictionary expects values of type list[str/int]")
-                for x in dictified[k]:
-                    if not isinstance(x, str) or not isinstance(x, int):
-                        raise ValueError("The dictionary expects values of type list[str/int]")
+                if not isinstance(dictified[k], StringProperty) or not isinstance(dictified[k], IntegerProperty):
+                    raise ValueError("The dictionary expects values of type str or int")
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index c1b6c373..e3aecd11 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -39,20 +39,27 @@ def test_dictionary_property_values_int():
         assert q.clean()
 
 def test_dictionary_property_values_stringlist():
-    p = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': ['123', '456']})
+    p = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': ['abc', 'def']})
     assert p.clean()
 
     q = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': '123'})
     with pytest.raises(ValueError):
         assert q.clean()
 
+    r = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': [123, 456]})
+    with pytest.raises(ValueError):
+        assert r.clean()
+
 def test_dictionary_property_values_list():
-    p = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': ['123', '456']})
+    p = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': 123})
     assert p.clean()
 
     q = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': '123'})
+    assert q.clean()
+
+    r = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': ['abc', 'def']})
     with pytest.raises(ValueError):
-        assert q.clean()
+        assert r.clean()
 
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'

From ee499637d4b1fa3372b73584b2ee8a9eb346ffc1 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 26 Mar 2024 17:57:53 -0400
Subject: [PATCH 017/132] handle schemas, process levels, fixed REs

---
 .../datastore/relational_db/input_creation.py |   17 +-
 .../datastore/relational_db/relational_db.py  |    3 +-
 .../datastore/relational_db/table_creation.py | 1231 +++++++++--------
 stix2/datastore/relational_db/utils.py        |    5 +-
 4 files changed, 655 insertions(+), 601 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 22a83fcf..be1f5596 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -13,12 +13,12 @@
 
 
 def single_value(p):
-    return not isinstance(
-        p, (
+    return not isinstance(
+        p, (
             EmbeddedObjectProperty,
             ListProperty,
-            DictionaryProperty,
-        ),
+            DictionaryProperty,
+        ),
     )
 
 
@@ -87,6 +87,7 @@ def generate_single_values(stix_object, properties, core_properties=[]):
 def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
     bindings = generate_single_values(item, item._properties)
     bindings["id"] = foreign_key_value
+    return bindings
 
 
 def generate_insert_for_dictionary(item, dictionary_table, foreign_key_value, value_types):
@@ -107,11 +108,11 @@ def generate_insert_for_dictionary(item, dictionary_table, foreign_key_value, va
         return [insert(dictionary_table).values(bindings)]
 
 
-def generate_insert_for_embedded_objects(type_name, values, foreign_key_value):
-    sql_bindings_tuples = list()
+def generate_insert_for_embedded_objects(table, type_name, values, foreign_key_value):
+    bindings = dict()
     for item in values:
-        sql_bindings_tuples.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
-    return sql_bindings_tuples
+        bindings.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
+    return [insert(table).values(bindings)]
 
 
 def generate_insert_for_hashes(hashes, hashes_table, foreign_key_value):
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index d05fb08e..ed2d8f90 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -9,6 +9,7 @@
 from stix2.datastore.relational_db.table_creation import (
     create_core_tables, generate_object_table,
 )
+from stix2.datastore.relational_db.utils import canonicalize_table_name
 from stix2.parsing import parse
 from stix2.v21.base import (
     _DomainObject, _Extension, _Observable, _RelationshipObject,
@@ -99,7 +100,7 @@ def __init__(
         self.tables = self._create_table_objects()
         self.tables_dictionary = dict()
         for t in self.tables:
-            self.tables_dictionary[t.name] = t
+            self.tables_dictionary[canonicalize_table_name(t.name, t.schema)] = t
 
         if instantiate_database:
             self._instantiate_database()
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 0f343c3f..3b7501c4 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -1,591 +1,640 @@
-# from collections import OrderedDict
-
-from sqlalchemy import (  # create_engine,; insert,
-    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text,
-)
-
-from stix2.datastore.relational_db.add_method import add_method
-from stix2.datastore.relational_db.utils import (
-    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
-)
-from stix2.properties import (
-    BinaryProperty, BooleanProperty, DictionaryProperty,
-    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
-    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
-    ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
-    TimestampProperty, TypeProperty,
-)
-from stix2.v21.common import KillChainPhase
-
-
-def aux_table_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
-    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
-        return True
-    else:
-        return False
-
-
-def derive_column_name(prop):
-    contained_property = prop.contained
-    if isinstance(contained_property, ReferenceProperty):
-        return "ref_id"
-    elif isinstance(contained_property, StringProperty):
-        return "value"
-
-
-def create_object_markings_refs_table(metadata, sco_or_sdo):
-    return create_ref_table(
-        metadata,
-        {"marking_definition"},
-        "object_marking_refs_" + sco_or_sdo,
-        "common.core_" + sco_or_sdo + ".id",
-        "common",
-        0,
-    )
-
-
-def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_name, auth_type=0):
-    columns = list()
-    columns.append(
-        Column(
-            "id",
-            Text,
-            ForeignKey(
-                foreign_key_name,
-                ondelete="CASCADE",
-            ),
-            nullable=False,
-        ),
-    )
-    columns.append(ref_column("ref_id", specifics, auth_type))
-    return Table(table_name, metadata, *columns, schema=schema_name)
-
-
-def create_hashes_table(name, metadata, schema_name, table_name):
-    columns = list()
-    columns.append(
-        Column(
-            "id",
-            Text,
-            ForeignKey(
-                canonicalize_table_name(table_name, schema_name) + ".id",
-                ondelete="CASCADE",
-            ),
-            nullable=False,
-        ),
-    )
-    columns.append(
-        Column(
-            "hash_name",
-            Text,
-            nullable=False,
-        ),
-    )
-    columns.append(
-        Column(
-            "hash_value",
-            Text,
-            nullable=False,
-        ),
-    )
-    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
-
-
-def create_granular_markings_table(metadata, sco_or_sdo):
-    return Table(
-        "granular_marking_" + sco_or_sdo,
-        metadata,
-        Column(
-            "id",
-            Text,
-            ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-        Column("lang", Text),
-        Column(
-            "marking_ref",
-            Text,
-            CheckConstraint(
-                "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-            ),
-        ),
-        Column(
-            "selectors",
-            ARRAY(Text),
-            CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
-            nullable=False,
-        ),
-        CheckConstraint(
-            """(lang IS NULL AND marking_ref IS NOT NULL)
-               OR
-               (lang IS NOT NULL AND marking_ref IS NULL)""",
-        ),
-        schema="common",
-    )
-
-
-def create_external_references_tables(metadata):
-    columns = [
-        Column(
-            "id",
-            Text,
-            ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
-            CheckConstraint(
-                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-        ),
-        Column("source_name", Text),
-        Column("description", Text),
-        Column("url", Text),
-        Column("external_id", Text),
-    ]
-    return [
-        Table("external_references", metadata, *columns, schema="common"),
-        #  create_hashes_table("hashes", metadata, "common", "external_references")
-    ]
-
-
-def create_core_table(metadata, schema_name):
-    columns = [
-        Column(
-            "id",
-            Text,
-            CheckConstraint(
-                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-        ),
-        Column("spec_version", Text, default="2.1"),
-        Column("object_marking_ref", ARRAY(Text)),
-    ]
-    if schema_name == "sdo":
-        sdo_columns = [
-            Column(
-                "created_by_ref",
-                Text,
-                CheckConstraint(
-                    "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                    # noqa: E131
-                ),
-            ),
-            Column("created", TIMESTAMP(timezone=True)),
-            Column("modified", TIMESTAMP(timezone=True)),
-            Column("revoked", Boolean),
-            Column("confidence", Integer),
-            Column("lang", Text),
-        ]
-        columns.extend(sdo_columns)
-    else:
-        columns.append(Column("defanged", Boolean, default=False)),
-    return Table(
-        "core_" + schema_name,
-        metadata,
-        *columns,
-        schema="common",
-    )
-
-
-@add_method(KillChainPhase)
-def determine_sql_type(self):
-    return None
-
-
-@add_method(Property)
-def generate_table_information(self, name, **kwargs):
-    pass
-
-
-@add_method(Property)
-def determine_sql_type(self):  # noqa: F811
-    pass
-
-
-@add_method(StringProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Text,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(StringProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Text
-
-
-@add_method(IntegerProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Integer,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(IntegerProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Integer
-
-
-@add_method(FloatProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Float,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(FloatProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Float
-
-
-@add_method(BooleanProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Boolean,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(BooleanProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Boolean
-
-
-@add_method(TypeProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Text,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(IDProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema") else "common.core_sco.id"
-    table_name = kwargs.get("table_name")
-    return Column(
-        name,
-        Text,
-        ForeignKey(foreign_key_column, ondelete="CASCADE"),
-        CheckConstraint(
-            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-        ),
-        primary_key=True,
-        nullable=not (self.required),
-    )
-
-
-@add_method(EnumProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    enum_re = "|".join(self.allowed)
-    return Column(
-        name,
-        Text,
-        CheckConstraint(
-            f"{name} ~ '^{enum_re}$'",
-        ),
-        nullable=not (self.required),
-    )
-
-
-@add_method(TimestampProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        TIMESTAMP(timezone=True),
-        # CheckConstraint(
-        #     f"{name} ~ '^{enum_re}$'"
-        # ),
-        nullable=not (self.required),
-    )
-
-
-@add_method(DictionaryProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
-    columns = list()
-
-    columns.append(
-        Column(
-            "id",
-            Text,
-            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
-        ),
-    )
-    columns.append(
-        Column(
-            "name",
-            Text,
-            nullable=False,
-        ),
-    )
-    if len(self.specifics) == 1:
-        if self.specifics[0] != "string_list":
-            columns.append(
-                Column(
-                    "value",
-                    Text if self.specifics[0] == "string" else Integer,
-                    nullable=False,
-                ),
-            )
-        else:
-            columns.append(
-                Column(
-                    "value",
-                    ARRAY(Text),
-                    nullable=False,
-                ),
-            )
-    else:
-        columns.append(
-            Column(
-                "string_value",
-                Text,
-            ),
-        )
-        columns.append(
-            Column(
-                "integer_value",
-                Integer,
-            ),
-        )
-    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
-
-
-@add_method(HashesProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
-    return [create_hashes_table(name, metadata, schema_name, table_name)]
-
-
-@add_method(HexProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        LargeBinary,
-        nullable=not (self.required),
-    )
-
-
-@add_method(BinaryProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    print("BinaryProperty not handled, yet")
-    return None
-
-
-@add_method(ExtensionsProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
-    columns = list()
-    columns.append(
-        Column(
-            "id",
-            Text,
-            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-    )
-    columns.append(
-        Column(
-            "ext_table_name",
-            Text,
-            nullable=False,
-        ),
-    )
-    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
-
-
-def ref_column(name, specifics, auth_type=0):
-    if specifics:
-        types = "|".join(specifics)
-        if auth_type == 0:
-            constraint = \
-                CheckConstraint(
-                    f"{name} ~ '^({types})" +
-                    "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                )
-        else:
-            constraint = \
-                CheckConstraint(
-                    f"(NOT({name} ~ '^({types})') AND ({name} ~ " +
-                    "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
-                )
-        return Column(name, Text, constraint)
-    else:
-        return Column(
-            name,
-            Text,
-            nullable=False,
-        )
-
-
-@add_method(ReferenceProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return ref_column(name, self.specifics, self.auth_type)
-
-
-@add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
-    return generate_object_table(self.type, metadata, schema_name, table_name, is_extension, True, is_list)
-
-
-@add_method(ObjectReferenceProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    table_name = kwargs.get('table_name')
-    raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
-
-
-def sub_objects(prop_class):
-    for name, prop in prop_class.type._properties.items():
-        if isinstance(prop, (HashesProperty, EmbeddedObjectProperty)):
-            return True
-    return False
-
-
-@add_method(ListProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
-    is_extension = kwargs.get('is_extension')
-    tables = list()
-    if isinstance(self.contained, ReferenceProperty):
-        return [
-            create_ref_table(
-                metadata,
-                self.contained.specifics,
-                canonicalize_table_name(table_name + "_" + name),
-                canonicalize_table_name(table_name, schema_name) + ".id",
-                "common",
-            ),
-        ]
-    elif isinstance(self.contained, EmbeddedObjectProperty):
-        columns = list()
-        columns.append(
-            Column(
-                "id",
-                Text,
-                ForeignKey(
-                    canonicalize_table_name(table_name, schema_name) + ".id",
-                    ondelete="CASCADE",
-                ),
-            ),
-        )
-        columns.append(
-            Column(
-                "ref_id",
-                Integer,
-                primary_key=True,
-                nullable=False,
-            ),
-        )
-        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
-        tables.extend(
-            self.contained.generate_table_information(
-                name,
-                metadata,
-                schema_name,
-                canonicalize_table_name(table_name + "_" + name, None),  # if sub_table_needed else canonicalize_table_name(table_name, None),
-                is_extension,
-                is_list=True,
-            ),
-        )
-        return tables
-    else:
-        if isinstance(self.contained, Property):
-            sql_type = self.contained.determine_sql_type()
-            if sql_type:
-                return Column(
-                    name,
-                    ARRAY(sql_type),
-                    nullable=not (self.required),
-                )
-
-
-def generate_object_table(
-    stix_object_class, metadata, schema_name, foreign_key_name=None,
-    is_extension=False, is_embedded_object=False, is_list=False,
-):
-    properties = stix_object_class._properties
-    if hasattr(stix_object_class, "_type"):
-        table_name = stix_object_class._type
-    else:
-        table_name = stix_object_class.__name__
-    if table_name.startswith("extension-definition"):
-        table_name = table_name[0:30]
-    core_properties = SDO_COMMON_PROPERTIES if schema_name else SCO_COMMON_PROPERTIES
-    columns = list()
-    tables = list()
-    for name, prop in properties.items():
-        if name == 'id' or name not in core_properties:
-            col = prop.generate_table_information(
-                name,
-                metadata=metadata,
-                schema_name=schema_name,
-                table_name=table_name,
-                is_extension=is_extension,
-                is_embedded_object=is_embedded_object,
-                is_list=is_list,
-            )
-            if col is not None and isinstance(col, Column):
-                columns.append(col)
-            if col is not None and isinstance(col, list):
-                tables.extend(col)
-    if (is_extension and not is_embedded_object):  # or (is_extension and is_embedded_object and is_list):
-        columns.append(
-            Column(
-                "id",
-                Text,
-                # no Foreign Key because it could be for different tables
-                primary_key=True,
-            ),
-        )
-    if foreign_key_name:
-        if is_extension or (is_embedded_object and is_list):
-            column = Column(
-                "id",
-                Integer if (is_embedded_object and is_list) else Text,
-                ForeignKey(
-                    canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if (is_embedded_object and is_list) else ".id"),
-                    ondelete="CASCADE",
-                ),
-            )
-        else:
-            column = Column(
-                "id",
-                Text,
-                ForeignKey(
-                    canonicalize_table_name(foreign_key_name, schema_name) + ".id",
-                    ondelete="CASCADE",
-                ),
-            )
-        columns.append(column)
-
-    all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
-    all_tables.extend(tables)
-    return all_tables
-
-
-def create_core_tables(metadata):
-    tables = [
-        create_core_table(metadata, "sdo"),
-        create_granular_markings_table(metadata, "sdo"),
-        create_core_table(metadata, "sco"),
-        create_granular_markings_table(metadata, "sco"),
-        create_object_markings_refs_table(metadata, "sdo"),
-        create_object_markings_refs_table(metadata, "sco"),
-    ]
-    tables.extend(create_external_references_tables(metadata))
-    return tables
+# from collections import OrderedDict
+
+from sqlalchemy import (  # create_engine,; insert,
+    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
+    Integer, LargeBinary, Table, Text,
+)
+
+from stix2.datastore.relational_db.add_method import add_method
+from stix2.datastore.relational_db.utils import (
+    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+)
+from stix2.properties import (
+    BinaryProperty, BooleanProperty, DictionaryProperty,
+    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
+    TimestampProperty, TypeProperty,
+)
+from stix2.v21.common import KillChainPhase
+
+
+def aux_table_property(prop, name, core_properties):
+    if isinstance(prop, ListProperty) and name not in core_properties:
+        contained_property = prop.contained
+        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
+    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
+        return True
+    else:
+        return False
+
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def create_object_markings_refs_table(metadata, sco_or_sdo):
+    return create_ref_table(
+        metadata,
+        {"marking_definition"},
+        "object_marking_refs_" + sco_or_sdo,
+        "common.core_" + sco_or_sdo + ".id",
+        "common",
+        0,
+    )
+
+
+def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_name, auth_type=0):
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(
+                foreign_key_name,
+                ondelete="CASCADE",
+            ),
+            nullable=False,
+        ),
+    )
+    columns.append(ref_column("ref_id", specifics, auth_type))
+    return Table(table_name, metadata, *columns, schema=schema_name)
+
+
+def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text, level=1):
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            key_type,
+            ForeignKey(
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                ondelete="CASCADE",
+            ),
+
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_value",
+            Text,
+            nullable=False,
+        ),
+    )
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
+
+
+def create_granular_markings_table(metadata, sco_or_sdo):
+    return Table(
+        "granular_marking_" + sco_or_sdo,
+        metadata,
+        Column(
+            "id",
+            Text,
+            ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+        Column("lang", Text),
+        Column(
+            "marking_ref",
+            Text,
+            CheckConstraint(
+                "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+            ),
+        ),
+        Column(
+            "selectors",
+            ARRAY(Text),
+            CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
+            nullable=False,
+        ),
+        CheckConstraint(
+            """(lang IS NULL AND marking_ref IS NOT NULL)
+               OR
+               (lang IS NOT NULL AND marking_ref IS NULL)""",
+        ),
+        schema="common",
+    )
+
+
+def create_external_references_tables(metadata):
+    columns = [
+        Column(
+            "id",
+            Text,
+            ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
+            CheckConstraint(
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+            ),
+            primary_key=True,
+        ),
+        Column("source_name", Text),
+        Column("description", Text),
+        Column("url", Text),
+        Column("external_id", Text),
+    ]
+    return [
+        Table("external_references", metadata, *columns, schema="common"),
+        #  create_hashes_table("hashes", metadata, "common", "external_references")
+    ]
+
+
+def create_core_table(metadata, schema_name):
+    columns = [
+        Column(
+            "id",
+            Text,
+            CheckConstraint(
+                "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+            ),
+            primary_key=True,
+        ),
+        Column("spec_version", Text, default="2.1"),
+        Column("object_marking_ref", ARRAY(Text)),
+    ]
+    if schema_name == "sdo":
+        sdo_columns = [
+            Column(
+                "created_by_ref",
+                Text,
+                CheckConstraint(
+                    "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",   # noqa: E131
+                ),
+            ),
+            Column("created", TIMESTAMP(timezone=True)),
+            Column("modified", TIMESTAMP(timezone=True)),
+            Column("revoked", Boolean),
+            Column("confidence", Integer),
+            Column("lang", Text),
+        ]
+        columns.extend(sdo_columns)
+    else:
+        columns.append(Column("defanged", Boolean, default=False)),
+    return Table(
+        "core_" + schema_name,
+        metadata,
+        *columns,
+        schema="common",
+    )
+
+
+@add_method(KillChainPhase)
+def determine_sql_type(self):
+    return None
+
+
+@add_method(Property)
+def generate_table_information(self, name, **kwargs):
+    pass
+
+
+@add_method(Property)
+def determine_sql_type(self):  # noqa: F811
+    pass
+
+
+@add_method(StringProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(StringProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Text
+
+
+@add_method(IntegerProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Integer,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(IntegerProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Integer
+
+
+@add_method(FloatProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Float,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(FloatProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Float
+
+
+@add_method(BooleanProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Boolean,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(BooleanProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Boolean
+
+
+@add_method(TypeProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(IDProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema") else "common.core_sco.id"
+    table_name = kwargs.get("table_name")
+    return Column(
+        name,
+        Text,
+        ForeignKey(foreign_key_column, ondelete="CASCADE"),
+        CheckConstraint(
+            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+        ),
+        primary_key=True,
+        nullable=not (self.required),
+    )
+
+
+@add_method(EnumProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    enum_re = "|".join(self.allowed)
+    return Column(
+        name,
+        Text,
+        CheckConstraint(
+            f"{name} ~ '^{enum_re}$'",
+        ),
+        nullable=not (self.required),
+    )
+
+
+@add_method(TimestampProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        TIMESTAMP(timezone=True),
+        # CheckConstraint(
+        #     f"{name} ~ '^{enum_re}$'"
+        # ),
+        nullable=not (self.required),
+    )
+
+
+@add_method(DictionaryProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
+    columns = list()
+
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
+        ),
+    )
+    columns.append(
+        Column(
+            "name",
+            Text,
+            nullable=False,
+        ),
+    )
+    if len(self.specifics) == 1:
+        if self.specifics[0] != "string_list":
+            columns.append(
+                Column(
+                    "value",
+                    Text if self.specifics[0] == "string" else Integer,
+                    nullable=False,
+                ),
+            )
+        else:
+            columns.append(
+                Column(
+                    "value",
+                    ARRAY(Text),
+                    nullable=False,
+                ),
+            )
+    else:
+        columns.append(
+            Column(
+                "string_value",
+                Text,
+            ),
+        )
+        columns.append(
+            Column(
+                "integer_value",
+                Integer,
+            ),
+        )
+    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
+
+
+@add_method(HashesProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
+    level = kwargs.get("level")
+    parent_table_name = kwargs.get("parent_table_name")
+    if kwargs.get("is_embedded_object"):
+        if not kwargs.get("is_list") or level == 0:
+            key_type = Text
+            # querky case where a property of an object is a single embedded objects
+            table_name = parent_table_name
+        else:
+            key_type = Integer
+    else:
+        key_type = Text
+    return [
+        create_hashes_table(
+            name,
+            metadata,
+            schema_name,
+            table_name,
+            key_type=key_type,
+            level=level,
+        ),
+    ]
+
+
+@add_method(HexProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        LargeBinary,
+        nullable=not self.required,
+    )
+
+
+@add_method(BinaryProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    print("BinaryProperty not handled, yet")
+    return None
+
+
+@add_method(ExtensionsProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "ext_table_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
+
+
+def ref_column(name, specifics, auth_type=0):
+    if specifics:
+        types = "|".join(specifics)
+        if auth_type == 0:
+            constraint = \
+                CheckConstraint(
+                    f"{name} ~ '^({types})" +
+                    "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                )
+        else:
+            constraint = \
+                CheckConstraint(
+                    f"(NOT({name} ~ '^({types})')) AND ({name} ~ " +
+                    "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
+                )
+        return Column(name, Text, constraint)
+    else:
+        return Column(
+            name,
+            Text,
+            nullable=False,
+        )
+
+
+@add_method(ReferenceProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return ref_column(name, self.specifics, self.auth_type)
+
+
+@add_method(EmbeddedObjectProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
+    level = kwargs.get("level")
+    return generate_object_table(
+        self.type, metadata, schema_name, table_name, is_extension, True, is_list,
+        parent_table_name=table_name, level=level+1 if is_list else level,
+    )
+
+
+@add_method(ObjectReferenceProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    table_name = kwargs.get('table_name')
+    raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
+
+
+def sub_objects(prop_class):
+    for name, prop in prop_class.type._properties.items():
+        if isinstance(prop, (HashesProperty, EmbeddedObjectProperty)):
+            return True
+    return False
+
+
+@add_method(ListProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
+    is_extension = kwargs.get('is_extension')
+    tables = list()
+    if isinstance(self.contained, ReferenceProperty):
+        return [
+            create_ref_table(
+                metadata,
+                self.contained.specifics,
+                canonicalize_table_name(table_name + "_" + name),
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                "common",
+            ),
+        ]
+    elif isinstance(self.contained, EmbeddedObjectProperty):
+        columns = list()
+        columns.append(
+            Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+                primary_key=True,
+            ),
+        )
+        columns.append(
+            Column(
+                "ref_id",
+                Integer,
+                primary_key=True,
+                nullable=False,
+            ),
+        )
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
+        tables.extend(
+            self.contained.generate_table_information(
+                name,
+                metadata,
+                schema_name,
+                canonicalize_table_name(table_name + "_" + name, None),  # if sub_table_needed else canonicalize_table_name(table_name, None),
+                is_extension,
+                parent_table_name=table_name,
+                is_list=True,
+                level=kwargs.get("level"),
+            ),
+        )
+        return tables
+    else:
+        if isinstance(self.contained, Property):
+            sql_type = self.contained.determine_sql_type()
+            if sql_type:
+                return Column(
+                    name,
+                    ARRAY(sql_type),
+                    nullable=not (self.required),
+                )
+
+
+def generate_object_table(
+    stix_object_class, metadata, schema_name, foreign_key_name=None,
+    is_extension=False, is_embedded_object=False, is_list=False, parent_table_name=None, level=0,
+):
+    properties = stix_object_class._properties
+    if hasattr(stix_object_class, "_type"):
+        table_name = stix_object_class._type
+    else:
+        table_name = stix_object_class.__name__
+    if table_name.startswith("extension-definition"):
+        table_name = table_name[0:30]
+    if parent_table_name:
+        table_name = parent_table_name + "_" + table_name
+    core_properties = SDO_COMMON_PROPERTIES if schema_name else SCO_COMMON_PROPERTIES
+    columns = list()
+    tables = list()
+    for name, prop in properties.items():
+        if name == 'id' or name not in core_properties:
+            col = prop.generate_table_information(
+                name,
+                metadata=metadata,
+                schema_name=schema_name,
+                table_name=table_name,
+                is_extension=is_extension,
+                is_embedded_object=is_embedded_object,
+                is_list=is_list,
+                level=level,
+                parent_table_name=parent_table_name,
+            )
+            if col is not None and isinstance(col, Column):
+                columns.append(col)
+            if col is not None and isinstance(col, list):
+                tables.extend(col)
+    if is_extension and not is_embedded_object:
+        columns.append(
+            Column(
+                "id",
+                Text,
+                # no Foreign Key because it could be for different tables
+                primary_key=True,
+            ),
+        )
+    if foreign_key_name:
+        if level == 0:
+            if is_extension and not is_embedded_object:
+                column = Column(
+                    "id",
+                    Text,
+                    ForeignKey(
+                        canonicalize_table_name(foreign_key_name, schema_name) + ".id",
+                        ondelete="CASCADE",
+                    ),
+                )
+            elif is_embedded_object:
+                column = Column(
+                    "id",
+                    Integer if is_list else Text,
+                    ForeignKey(
+                        canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if is_list else ".id"),
+                        ondelete="CASCADE",
+                    ),
+                )
+        elif level > 0 and is_embedded_object:
+            column = Column(
+                "id",
+                Integer if (is_embedded_object and is_list) else Text,
+                ForeignKey(
+                    canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if (is_embedded_object and is_list) else ".id"),
+                    ondelete="CASCADE",
+                ),
+                primary_key=True,
+            )
+        else:
+            column = Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(foreign_key_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+            )
+        columns.append(column)
+
+    all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
+    all_tables.extend(tables)
+    return all_tables
+
+
+def create_core_tables(metadata):
+    tables = [
+        create_core_table(metadata, "sdo"),
+        create_granular_markings_table(metadata, "sdo"),
+        create_core_table(metadata, "sco"),
+        create_granular_markings_table(metadata, "sco"),
+        create_object_markings_refs_table(metadata, "sdo"),
+        create_object_markings_refs_table(metadata, "sco"),
+    ]
+    tables.extend(create_external_references_tables(metadata))
+    return tables
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index e908ce89..e45897ac 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -1,3 +1,5 @@
+import inflection
+
 # Helps us know which data goes in core, and which in a type-specific table.
 SCO_COMMON_PROPERTIES = {
     "id",
@@ -32,4 +34,5 @@ def canonicalize_table_name(table_name, schema_name=None):
         full_name = schema_name + "." + table_name
     else:
         full_name = table_name
-    return full_name.replace("-", "_")
+    full_name = full_name.replace("-", "_")
+    return inflection.underscore(full_name)

From 9a1663f9a0a39a35b52092dda49d924344b6df41 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 27 Mar 2024 12:38:48 -0400
Subject: [PATCH 018/132] fix schema name for subtables, composite primary
 keys, RelationalRBStore stub

---
 .../datastore/relational_db/relational_db.py  | 49 ++++++++++++++++++-
 .../datastore/relational_db/table_creation.py |  5 +-
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index ed2d8f90..f3027da4 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -2,7 +2,7 @@
 from sqlalchemy.schema import CreateSchema, CreateTable
 
 from stix2.base import _STIXBase
-from stix2.datastore import DataSink
+from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.relational_db.input_creation import (
     generate_insert_for_object,
 )
@@ -62,6 +62,42 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
         store.insert_object(stix_obj)
 
 
+class RelationalDBStore(DataStoreMixin):
+    """Interface to a file directory of STIX objects.
+
+    FileSystemStore is a wrapper around a paired FileSystemSink
+    and FileSystemSource.
+
+    Args:
+        stix_dir (str): path to directory of STIX objects
+        allow_custom (bool): whether to allow custom STIX content to be
+            pushed/retrieved. Defaults to True for FileSystemSource side
+            (retrieving data) and False for FileSystemSink
+            side(pushing data). However, when parameter is supplied, it
+            will be applied to both FileSystemSource and FileSystemSink.
+        bundlify (bool): whether to wrap objects in bundles when saving
+            them. Default: False.
+        encoding (str): The encoding to use when reading a file from the
+            filesystem.
+
+    Attributes:
+        source (FileSystemSource): FileSystemSource
+        sink (FileSystemSink): FileSystemSink
+
+    """
+    def __init__(self, database_connection_url, allow_custom=None, encoding='utf-8'):
+        if allow_custom is None:
+            allow_custom_source = True
+            allow_custom_sink = False
+        else:
+            allow_custom_sink = allow_custom_source = allow_custom
+
+        super(RelationalDBStore, self).__init__(
+            source=RelationalDBSource(database_connection_url, allow_custom=allow_custom_source, encoding=encoding),
+            sink=RelationalDBSink(database_connection_url, allow_custom=allow_custom_sink),
+        )
+
+
 class RelationalDBSink(DataSink):
     """Interface for adding/pushing STIX objects to an in-memory dictionary.
 
@@ -150,3 +186,14 @@ def insert_object(self, stix_object):
                 print("executing: ", stmt)
                 trans.execute(stmt)
             trans.commit()
+
+
+class RelationalDBSource(DataSource):
+    def get(self, stix_id, version=None, _composite_filters=None):
+        pass
+
+    def all_versions(self, stix_id, version=None, _composite_filters=None):
+        pass
+
+    def query(self, query=None):
+        pass
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 3b7501c4..d22001ba 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -490,7 +490,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 self.contained.specifics,
                 canonicalize_table_name(table_name + "_" + name),
                 canonicalize_table_name(table_name, schema_name) + ".id",
-                "common",
+                schema_name,
             ),
         ]
     elif isinstance(self.contained, EmbeddedObjectProperty):
@@ -502,8 +502,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 ForeignKey(
                     canonicalize_table_name(table_name, schema_name) + ".id",
                     ondelete="CASCADE",
-                ),
-                primary_key=True,
+                )
             ),
         )
         columns.append(

From 5f2303325a4e0a81cfe0d9f6ba7fc7f210fff79e Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 27 Mar 2024 14:57:48 -0400
Subject: [PATCH 019/132] added MetaObject

---
 stix2/base.py                                   | 4 ++++
 stix2/datastore/relational_db/input_creation.py | 1 +
 stix2/datastore/relational_db/relational_db.py  | 4 +++-
 stix2/datastore/relational_db/table_creation.py | 2 +-
 stix2/v20/base.py                               | 7 ++++++-
 stix2/v20/common.py                             | 4 ++--
 stix2/v21/base.py                               | 7 ++++++-
 stix2/v21/common.py                             | 8 ++++----
 8 files changed, 27 insertions(+), 10 deletions(-)

diff --git a/stix2/base.py b/stix2/base.py
index 3ff01d98..7d97b8a4 100644
--- a/stix2/base.py
+++ b/stix2/base.py
@@ -470,6 +470,10 @@ def _check_object_constraints(self):
         self._check_at_least_one_property()
 
 
+class _MetaObject(_STIXBase):
+    pass
+
+
 def _choose_one_hash(hash_dict):
     if "MD5" in hash_dict:
         return {"MD5": hash_dict["MD5"]}
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index be1f5596..1f46b992 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -2,6 +2,7 @@
 
 from sqlalchemy import insert
 
+from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
 )
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index f3027da4..e27562af 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -12,7 +12,7 @@
 from stix2.datastore.relational_db.utils import canonicalize_table_name
 from stix2.parsing import parse
 from stix2.v21.base import (
-    _DomainObject, _Extension, _Observable, _RelationshipObject,
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
 )
 
 
@@ -158,6 +158,8 @@ def _create_table_objects(self):
             tables.extend(new_tables)
         for stix_class in _get_all_subclasses(_Observable):
             tables.extend(generate_object_table(stix_class, self.metadata, "sco"))
+        for stix_class in _get_all_subclasses(_MetaObject):
+            tables.extend(generate_object_table(stix_class, self.metadata, "common"))
         for stix_class in _get_all_subclasses(_Extension):
             if stix_class.extension_type not in ["new-sdo", "new-sco", "new-sro"]:
                 if hasattr(stix_class, "_applies_to"):
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index d22001ba..86660289 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -502,7 +502,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 ForeignKey(
                     canonicalize_table_name(table_name, schema_name) + ".id",
                     ondelete="CASCADE",
-                )
+                ),
             ),
         )
         columns.append(
diff --git a/stix2/v20/base.py b/stix2/v20/base.py
index b5437ca6..45698e3e 100644
--- a/stix2/v20/base.py
+++ b/stix2/v20/base.py
@@ -1,7 +1,8 @@
 """Base classes for STIX 2.0 type definitions."""
 
 from ..base import (
-    _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase,
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
+    _STIXBase,
 )
 
 
@@ -23,3 +24,7 @@ class _DomainObject(_DomainObject, _STIXBase20):
 
 class _RelationshipObject(_RelationshipObject, _STIXBase20):
     pass
+
+
+class _MetaObject(_MetaObject, _STIXBase20):
+    pass
diff --git a/stix2/v20/common.py b/stix2/v20/common.py
index feaa3efc..b5a9ca5a 100644
--- a/stix2/v20/common.py
+++ b/stix2/v20/common.py
@@ -11,7 +11,7 @@
     SelectorProperty, StringProperty, TimestampProperty, TypeProperty,
 )
 from ..utils import NOW, _get_dict
-from .base import _STIXBase20
+from .base import _MetaObject, _STIXBase20
 from .vocab import HASHING_ALGORITHM
 
 
@@ -111,7 +111,7 @@ def clean(self, value, allow_custom=False):
             raise ValueError("must be a Statement, TLP Marking or a registered marking.")
 
 
-class MarkingDefinition(_STIXBase20, _MarkingsMixin):
+class MarkingDefinition(_MetaObject, _MarkingsMixin):
     """For more detailed information on this object's properties, see
     `the STIX 2.0 specification <http://docs.oasis-open.org/cti/stix/v2.0/cs01/part1-stix-core/stix-v2.0-cs01-part1-stix-core.html#_Toc496709284>`__.
     """
diff --git a/stix2/v21/base.py b/stix2/v21/base.py
index 5f14f1bb..fb3a1966 100644
--- a/stix2/v21/base.py
+++ b/stix2/v21/base.py
@@ -1,7 +1,8 @@
 """Base classes for STIX 2.1 type definitions."""
 
 from ..base import (
-    _DomainObject, _Extension, _Observable, _RelationshipObject, _STIXBase,
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
+    _STIXBase,
 )
 
 
@@ -40,3 +41,7 @@ class _DomainObject(_DomainObject, _STIXBase21):
 
 class _RelationshipObject(_RelationshipObject, _STIXBase21):
     pass
+
+
+class _MetaObject(_MetaObject, _STIXBase21):
+    pass
diff --git a/stix2/v21/common.py b/stix2/v21/common.py
index d18bcc03..218070ff 100644
--- a/stix2/v21/common.py
+++ b/stix2/v21/common.py
@@ -14,7 +14,7 @@
     TypeProperty,
 )
 from ..utils import NOW, _get_dict
-from .base import _STIXBase21
+from .base import _MetaObject, _STIXBase21
 from .vocab import EXTENSION_TYPE, HASHING_ALGORITHM
 
 
@@ -79,7 +79,7 @@ def _check_object_constraints(self):
         self._check_at_least_one_property(['lang', 'marking_ref'])
 
 
-class LanguageContent(_STIXBase21):
+class LanguageContent(_MetaObject):
     """For more detailed information on this object's properties, see
     `the STIX 2.1 specification <https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_z9r1cwtu8jja>`__.
     """
@@ -107,7 +107,7 @@ class LanguageContent(_STIXBase21):
     ])
 
 
-class ExtensionDefinition(_STIXBase21):
+class ExtensionDefinition(_MetaObject):
     """For more detailed information on this object's properties, see
     `the STIX 2.1 specification <https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_32j232tfvtly>`__.
     """
@@ -190,7 +190,7 @@ def clean(self, value, allow_custom=False):
             raise ValueError("must be a Statement, TLP Marking or a registered marking.")
 
 
-class MarkingDefinition(_STIXBase21, _MarkingsMixin):
+class MarkingDefinition(_MetaObject, _MarkingsMixin):
     """For more detailed information on this object's properties, see
     `the STIX 2.1 specification <https://docs.oasis-open.org/cti/stix/v2.1/os/stix-v2.1-os.html#_k5fndj2c7c1k>`__.
     """

From 337f095a55155babc1abf93d9c9f475934417783 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Thu, 28 Mar 2024 13:42:28 -0400
Subject: [PATCH 020/132] 3/28 edits

---
 stix2/properties.py | 54 ++++++++++++++++++++++++++++-----------------
 1 file changed, 34 insertions(+), 20 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 2ddeadd6..e44e17a0 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -391,18 +391,18 @@ def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
         if not valid_types:
-            valid_types = ["string"]
-        elif not isinstance(valid_types, ListProperty):
-            valid_types = [valid_types]
+            valid_types = StringProperty
+        # elif not isinstance(valid_types, ListProperty):
+        #     valid_types = [valid_types]
         
-        if 'string_list' in valid_types and len(valid_types) > 1:
-            raise ValueError("'string_list' cannot be combined with other types in a list.")
+        # if 'string_list' in valid_types and len(valid_types) > 1:
+        #     raise ValueError("'string_list' cannot be combined with other types in a list.")
 
-        for type_ in valid_types:
-            if type_ not in ("string", "integer", "string_list"):
-                raise ValueError("The value of a dictionary key cannot be ", type_)
+        # for type_ in valid_types:
+        #     if type_ not in ("string", "integer", "string_list"):
+        #         raise ValueError("The value of a dictionary key cannot be ", type_)
 
-        self.specifics = valid_types
+        self.valid_types = valid_types
 
         super(DictionaryProperty, self).__init__(**kwargs)
 
@@ -430,18 +430,32 @@ def clean(self, value, allow_custom=False):
                 )
                 raise DictionaryKeyError(k, msg)
             
-            if "string" in valid_types:
-                if not isinstance(dictified[k], StringProperty):
-                    raise ValueError("The dictionary expects values of type str")
-            elif "integer" in valid_types:
-                if not isinstance(dictified[k], IntegerProperty):
-                    raise ValueError("The dictionary expects values of type int")
-            elif "string_list" in valid_types:
-                if not isinstance(dictified[k], ListProperty(StringProperty)):
-                    raise ValueError("The dictionary expects values of type list[str]")
+            # if "string" in valid_types:
+            #     if not isinstance(dictified[k], StringProperty):
+            #         raise ValueError("The dictionary expects values of type str")
+            # elif "integer" in valid_types:
+            #     if not isinstance(dictified[k], IntegerProperty):
+            #         raise ValueError("The dictionary expects values of type int")
+            # elif "string_list" in valid_types:
+            #     if not isinstance(dictified[k], ListProperty(StringProperty)):
+            #         raise ValueError("The dictionary expects values of type list[str]")
+            # else:
+            #     if not isinstance(dictified[k], StringProperty) or not isinstance(dictified[k], IntegerProperty):
+            #         raise ValueError("The dictionary expects values of type str or int")
+
+            simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
+
+            if self.valid_types in simple_type:
+                self.valid_types.clean(dict[k])
+            elif isinstance(dictified[k], ListProperty()):
+                list_type = dictified[k].contained
+                if list_type in simple_type:
+                    for x in dictified[k]:
+                        list_type.clean(x)
+                else:
+                    raise ValueError("Dictionary Property does not support lists of type: ", list_type)
             else:
-                if not isinstance(dictified[k], StringProperty) or not isinstance(dictified[k], IntegerProperty):
-                    raise ValueError("The dictionary expects values of type str or int")
+                raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")

From 6c56457deed41387b58cab3da36db274d0c6cacd Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Thu, 28 Mar 2024 14:10:17 -0400
Subject: [PATCH 021/132] Test update

---
 stix2/properties.py               | 31 ++++++++++++++++++-------------
 stix2/test/v21/test_properties.py | 22 +++++++++++-----------
 2 files changed, 29 insertions(+), 24 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index e44e17a0..7b81bb88 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -391,7 +391,7 @@ def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
         if not valid_types:
-            valid_types = StringProperty
+            valid_types = [StringProperty]
         # elif not isinstance(valid_types, ListProperty):
         #     valid_types = [valid_types]
         
@@ -444,18 +444,23 @@ def clean(self, value, allow_custom=False):
             #         raise ValueError("The dictionary expects values of type str or int")
 
             simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
-
-            if self.valid_types in simple_type:
-                self.valid_types.clean(dict[k])
-            elif isinstance(dictified[k], ListProperty()):
-                list_type = dictified[k].contained
-                if list_type in simple_type:
-                    for x in dictified[k]:
-                        list_type.clean(x)
-                else:
-                    raise ValueError("Dictionary Property does not support lists of type: ", list_type)
-            else:
-                raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
+            clear = False
+            for type in self.valid_types:
+                if type in simple_type:
+                    try:
+                        self.valid_types.clean(dict[k])
+                    except ValueError:
+                        continue
+                    clear = True
+                elif isinstance(dictified[k], ListProperty()):
+                    list_type = dictified[k].contained
+                    if list_type in simple_type:
+                        for x in dictified[k]:
+                            list_type.clean(x)
+                    else:
+                        raise ValueError("Dictionary Property does not support lists of type: ", list_type)
+                if not clear:
+                    raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index e3aecd11..4aeeb8bc 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -8,7 +8,7 @@
 from stix2.properties import (
     DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty,
     HashesProperty, IDProperty, ListProperty, ObservableProperty,
-    ReferenceProperty, STIXObjectProperty,
+    ReferenceProperty, STIXObjectProperty, StringProperty, IntegerProperty
 )
 from stix2.v21.common import MarkingProperty
 
@@ -23,41 +23,41 @@ def test_dictionary_property():
         p.clean({})
 
 def test_dictionary_property_values_str():
-    p = DictionaryProperty(valid_types="string", spec_version='2.1', value={'x': '123'})
+    p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1', value={'x': '123'})
     assert p.clean()
 
-    q = DictionaryProperty(valid_types="string", spec_version='2.1', value={'x': 123})
+    q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1', value={'x': 123})
     with pytest.raises(ValueError):
         assert q.clean()
 
 def test_dictionary_property_values_int():
-    p = DictionaryProperty(valid_types="integer", spec_version='2.1', value={'x': 123})
+    p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1', value={'x': 123})
     assert p.clean()
 
-    q = DictionaryProperty(valid_types="integer", spec_version='2.1', value={'x': '123'})
+    q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1', value={'x': '123'})
     with pytest.raises(ValueError):
         assert q.clean()
 
 def test_dictionary_property_values_stringlist():
-    p = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': ['abc', 'def']})
+    p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1', value={'x': ['abc', 'def']})
     assert p.clean()
 
-    q = DictionaryProperty(valid_types="string_list", spec_version='2.1', value={'x': '123'})
+    q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1', value={'x': '123'})
     with pytest.raises(ValueError):
         assert q.clean()
 
-    r = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': [123, 456]})
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': [123, 456]})
     with pytest.raises(ValueError):
         assert r.clean()
 
 def test_dictionary_property_values_list():
-    p = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': 123})
+    p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': 123})
     assert p.clean()
 
-    q = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': '123'})
+    q = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': '123'})
     assert q.clean()
 
-    r = DictionaryProperty(valid_types=['string', 'integer'], spec_version='2.1', value={'x': ['abc', 'def']})
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': ['abc', 'def']})
     with pytest.raises(ValueError):
         assert r.clean()
 

From cb2aacb1024ea0c07698275ceebea0ded1ef1416 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 28 Mar 2024 21:35:23 -0400
Subject: [PATCH 022/132] lots of tweaks for tabe creation, input_creation
 intial success

---
 .../datastore/relational_db/input_creation.py | 282 ++++++++++--------
 .../datastore/relational_db/table_creation.py |  21 +-
 2 files changed, 161 insertions(+), 142 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 1f46b992..facc12e6 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -7,126 +7,153 @@
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
 )
 from stix2.properties import (
-    DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
-    ExtensionsProperty, FloatProperty, IntegerProperty, ListProperty,
-    ReferenceProperty, StringProperty,
+    BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
+    ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    Property, ReferenceProperty, StringProperty, TimestampProperty,
 )
 
 
-def single_value(p):
-    return not isinstance(
-        p, (
-            EmbeddedObjectProperty,
-            ListProperty,
-            DictionaryProperty,
-        ),
-    )
+@add_method(Property)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    pass
 
 
-def table_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
-    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
-        return True
-    else:
-        return False
+@add_method(BinaryProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
 
-def embedded_object_list_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return isinstance(contained_property, EmbeddedObjectProperty)
-    else:
-        return False
+@add_method(BooleanProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
 
-def array_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty, EnumProperty))
-    else:
-        return False
+@add_method(DictionaryProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    bindings = {"id": stix_object["id"]}
+    table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name,
+                                                                schema_name)]
+    for idx, (name, value) in enumerate(stix_object.items()):
+        name_binding = f"name{idx}"
+        if len(self.value_types) == 1:
+            value_binding = f"value{idx}"
+        elif isinstance(value, int):
+            value_binding = f"integer_value{idx}"
+        else:
+            value_binding = f"string_value{idx}"
 
+        bindings[name_binding] = name
+        bindings[value_binding] = value
 
-def derive_column_name(prop):
-    contained_property = prop.contained
-    if isinstance(contained_property, ReferenceProperty):
-        return "ref_id"
-    elif isinstance(contained_property, StringProperty):
-        return "value"
+        return [insert(table).values(bindings)]
 
 
-def generate_insert_for_array_in_table(table, values, foreign_key_value):
+@add_method(EmbeddedObjectProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return generate_insert_for_embedded_object(data_sink, stix_object[name], name, schema_name)
 
-    bindings = {
-        "id": foreign_key_value,
-    }
 
-    for idx, item in enumerate(values):
-        item_binding_name = f"item{idx}"
+@add_method(EnumProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
-        bindings[item_binding_name] = item
 
-    return [insert(table).values(bindings)]
+@add_method(FloatProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
 
-def generate_single_values(stix_object, properties, core_properties=[]):
-    bindings = OrderedDict()
-    for name, prop in properties.items():
-        if (
-            single_value(prop) and (name == 'id' or name not in core_properties) or
-            array_property(prop, name, core_properties)
-        ):
-            if name in stix_object and name != "type":
-                bindings[name] = stix_object[name] if not array_property(prop, name, core_properties) else "{" + ",".join(
-                    ['"' + x + '"' for x in stix_object[name]],
-                ) + "}"
-    return bindings
+@add_method(HexProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
 
-def generate_insert_for_embedded_object(type_name, item, foreign_key_value):
-    bindings = generate_single_values(item, item._properties)
-    bindings["id"] = foreign_key_value
-    return bindings
+def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name):
+    bindings = {"id": stix_object["id"]}
+    table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name,
+                                                                schema_name)]
 
+    for idx, (hash_name, hash_value) in enumerate(stix_object["hashes".items()]):
+        hash_name_binding_name = "hash_name" + str(idx)
+        hash_value_binding_name = "hash_value" + str(idx)
 
-def generate_insert_for_dictionary(item, dictionary_table, foreign_key_value, value_types):
-    bindings = {"id": foreign_key_value}
+        bindings[hash_name_binding_name] = hash_name
+        bindings[hash_value_binding_name] = hash_value
 
-    for idx, (name, value) in enumerate(item.items()):
-        name_binding = f"name{idx}"
-        if len(value_types) == 1:
-            value_binding = f"value{idx}"
-        elif isinstance(value, int):
-            value_binding = f"integer_value{idx}"
-        else:
-            value_binding = f"string_value{idx}"
+    return [insert(table).values(bindings)]
 
-        bindings[name_binding] = name
-        bindings[value_binding] = value
 
-        return [insert(dictionary_table).values(bindings)]
+@add_method(HashesProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name)
+
+@add_method(IDProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
+
+
+@add_method(IntegerProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
+
+
+@add_method(ListProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    if isinstance(self.contained, ReferenceProperty):
+        insert_statements = list()
+
+        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
+        for idx, item in enumerate(stix_object[name]):
+            bindings = {
+                "id": stix_object["id"],
+                "ref_id": item
+            }
+            insert_statements.append(insert(table).values(bindings))
+        return insert_statements
+    elif isinstance(self.contained, EmbeddedObjectProperty):
+        insert_statements = list()
+        for value in stix_object[name]:
+            insert_statements.extend(generate_insert_for_embedded_object(data_sink, value, table_name + "_" + name, None))
+            table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, None)]
+            insert_statements.append(insert(table).values({"id": stix_object["id"], "ref_id": 1}))
+        return insert_statements
+    else:
+        return {name: stix_object[name]}
 
 
-def generate_insert_for_embedded_objects(table, type_name, values, foreign_key_value):
-    bindings = dict()
-    for item in values:
-        bindings.extend(generate_insert_for_embedded_object(type_name, item, foreign_key_value))
-    return [insert(table).values(bindings)]
+@add_method(ReferenceProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
 
-def generate_insert_for_hashes(hashes, hashes_table, foreign_key_value):
-    bindings = {"id": foreign_key_value}
+@add_method(StringProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
-    for idx, (hash_name, hash_value) in enumerate(hashes.items()):
-        hash_name_binding_name = "hash_name" + str(idx)
-        hash_value_binding_name = "hash_value" + str(idx)
 
-        bindings[hash_name_binding_name] = hash_name
-        bindings[hash_value_binding_name] = hash_value
+@add_method(TimestampProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+    return {name: stix_object[name]}
 
-    return [insert(hashes_table).values(bindings)]
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def generate_insert_for_array_in_table(table, values, foreign_key_value):
+    insert_statements = list()
+    for idx, item in enumerate(values):
+        bindings = {
+            "id": foreign_key_value,
+            "ref_id": item
+        }
+        insert_statements.append(insert(table).values(bindings))
+
+    return insert_statements
 
 
 def generate_insert_for_external_references(data_sink, stix_object):
@@ -141,14 +168,8 @@ def generate_insert_for_external_references(data_sink, stix_object):
         insert_statements.append(er_insert_statement)
 
         if "hashes" in er:
-            hashes_table = data_sink.tables_dictionary[canonicalize_table_name("external_references_hashes", "sdo")]
             insert_statements.extend(
-                generate_insert_for_hashes(
-                    er["hashes"],
-                    hashes_table,
-                    stix_object["id"],
-                ),
-            )
+                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"))
 
     return insert_statements
 
@@ -196,7 +217,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
     for prop_name, value in stix_object.items():
 
         if prop_name in core_properties:
-            # stored in separate tables, skip here
+            # stored in separate tables below, skip here
             if prop_name not in {"object_marking_refs", "granular_markings", "external_references", "type"}:
                 core_bindings[prop_name] = value
 
@@ -231,8 +252,30 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
     return insert_statements
 
 
-def generate_insert_for_object(data_sink, stix_object, schema_name, foreign_key_value=None):
+def generate_insert_for_embedded_object(data_sink, stix_object, type_name, schema_name):
     insert_statements = list()
+    bindings = dict()
+    table_name = canonicalize_table_name(type_name, schema_name)
+    object_table = data_sink.tables_dictionary[table_name]
+    sub_insert_statements = list()
+    for name, prop in stix_object._properties.items():
+            if name in stix_object:
+                result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
+                if isinstance(result,dict):
+                    bindings.update(result)
+                elif isinstance(result,list):
+                    sub_insert_statements.extend(result)
+                else:
+                    raise(ValueError(result))
+
+    insert_statements.append(insert(object_table).values(bindings))
+    insert_statements.extend(sub_insert_statements)
+    return insert_statements
+
+
+def generate_insert_for_object(data_sink, stix_object, schema_name):
+    insert_statements = list()
+    bindings = dict()
     stix_id = stix_object["id"]
     if schema_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES
@@ -241,45 +284,28 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, foreign_key_
     type_name = stix_object["type"]
     table_name = canonicalize_table_name(type_name, schema_name)
     object_table = data_sink.tables_dictionary[table_name]
-    properties = stix_object._properties
     insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
 
-    bindings = generate_single_values(stix_object, properties, core_properties)
-    object_insert_statement = insert(object_table).values(bindings)
-    insert_statements.append(object_insert_statement)
-
+    sub_insert_statements = list()
     for name, prop in stix_object._properties.items():
-        if isinstance(prop, DictionaryProperty) and not name == "extensions":
-            dictionary_table_name = canonicalize_table_name(type_name + "_" + name, schema_name)
-            dictionary_table = data_sink.tables_dictionary[dictionary_table_name]
-            insert_statements.extend(generate_insert_for_dictionary(stix_object[name], dictionary_table, stix_id))
+        if name == 'id' or name not in core_properties:
+            if name in stix_object:
+                result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
+                if isinstance(result,dict):
+                    bindings.update(result)
+                elif isinstance(result,list):
+                    sub_insert_statements.extend(result)
+                else:
+                    raise(ValueError(result))
+
+    insert_statements.append(insert(object_table).values(bindings))
+    insert_statements.extend(sub_insert_statements)
 
     if "external_references" in stix_object:
-        insert_statements.extend(generate_insert_for_external_references(data_sink, stix_object, "sdo"))
+        insert_statements.extend(generate_insert_for_external_references(data_sink, stix_object))
 
     if "extensions" in stix_object:
         for ex in stix_object["extensions"]:
             insert_statements.extend(generate_insert_for_object(data_sink, ex, schema_name, stix_id))
-    for name, prop in properties.items():
-        if table_property(prop, name, core_properties):
-            if name in stix_object:
-                if embedded_object_list_property(prop, name, core_properties):
-                    insert_statements.extend(
-                        generate_insert_for_embedded_objects(
-                            name,
-                            stix_object[name],
-                            stix_object["id"],
-                        ),
-                    )
-                elif isinstance(prop, ExtensionsProperty):
-                    pass
-                else:
-                    insert_statements.extend(
-                        generate_insert_for_array_in_table(
-                            stix_object["type"],
-                            name,
-                            stix_object[name],
-                            properties[name],
-                            stix_object["id"], ),
-                    )
+
     return insert_statements
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 86660289..d9efeba3 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -40,7 +40,7 @@ def derive_column_name(prop):
 def create_object_markings_refs_table(metadata, sco_or_sdo):
     return create_ref_table(
         metadata,
-        {"marking_definition"},
+        {"marking-definition"},
         "object_marking_refs_" + sco_or_sdo,
         "common.core_" + sco_or_sdo + ".id",
         "common",
@@ -137,8 +137,7 @@ def create_external_references_tables(metadata):
             ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
             CheckConstraint(
                 "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-            ),
-            primary_key=True,
+            )
         ),
         Column("source_name", Text),
         Column("description", Text),
@@ -161,8 +160,7 @@ def create_core_table(metadata, schema_name):
             ),
             primary_key=True,
         ),
-        Column("spec_version", Text, default="2.1"),
-        Column("object_marking_ref", ARRAY(Text)),
+        Column("spec_version", Text, default="2.1")
     ]
     if schema_name == "sdo":
         sdo_columns = [
@@ -178,6 +176,7 @@ def create_core_table(metadata, schema_name):
             Column("revoked", Boolean),
             Column("confidence", Integer),
             Column("lang", Text),
+            Column("labels", ARRAY(Text))
         ]
         columns.extend(sdo_columns)
     else:
@@ -277,7 +276,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 @add_method(IDProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema") else "common.core_sco.id"
+    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema_name") else "common.core_sco.id"
     table_name = kwargs.get("table_name")
     return Column(
         name,
@@ -472,13 +471,6 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
-def sub_objects(prop_class):
-    for name, prop in prop_class.type._properties.items():
-        if isinstance(prop, (HashesProperty, EmbeddedObjectProperty)):
-            return True
-    return False
-
-
 @add_method(ListProperty)
 def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     is_extension = kwargs.get('is_extension')
@@ -547,11 +539,12 @@ def generate_object_table(
         table_name = stix_object_class._type
     else:
         table_name = stix_object_class.__name__
+    # avoid long table names
     if table_name.startswith("extension-definition"):
         table_name = table_name[0:30]
     if parent_table_name:
         table_name = parent_table_name + "_" + table_name
-    core_properties = SDO_COMMON_PROPERTIES if schema_name else SCO_COMMON_PROPERTIES
+    core_properties = SDO_COMMON_PROPERTIES if schema_name=="sdo" else SCO_COMMON_PROPERTIES
     columns = list()
     tables = list()
     for name, prop in properties.items():

From 79af97142ad26249fe1f4748e3c025491c3cfb51 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 28 Mar 2024 21:56:52 -0400
Subject: [PATCH 023/132] flaky

---
 .../datastore/relational_db/input_creation.py | 98 +++++++++++--------
 .../datastore/relational_db/table_creation.py |  8 +-
 2 files changed, 60 insertions(+), 46 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index facc12e6..dc6ab583 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -1,4 +1,3 @@
-from collections import OrderedDict
 
 from sqlalchemy import insert
 
@@ -7,32 +6,37 @@
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
 )
 from stix2.properties import (
-    BinaryProperty, BooleanProperty, DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
-    ExtensionsProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    BinaryProperty, BooleanProperty, DictionaryProperty,
+    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
     Property, ReferenceProperty, StringProperty, TimestampProperty,
 )
 
 
 @add_method(Property)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     pass
 
 
 @add_method(BinaryProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(BooleanProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(DictionaryProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     bindings = {"id": stix_object["id"]}
-    table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name,
-                                                                schema_name)]
+    table = data_sink.tables_dictionary[
+        canonicalize_table_name(
+            table_name + "_" + name,
+            schema_name,
+        )
+    ]
     for idx, (name, value) in enumerate(stix_object.items()):
         name_binding = f"name{idx}"
         if len(self.value_types) == 1:
@@ -49,29 +53,38 @@ def generate_insert_information(self, data_sink, name, stix_object, table_name,
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return generate_insert_for_embedded_object(data_sink, stix_object[name], name, schema_name)
 
 
 @add_method(EnumProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
+@add_method(ExtensionsProperty)
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
+    pass
+
+
 @add_method(FloatProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(HexProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name):
     bindings = {"id": stix_object["id"]}
-    table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name,
-                                                                schema_name)]
+    table = data_sink.tables_dictionary[
+        canonicalize_table_name(
+            table_name + "_" + name,
+            schema_name,
+        )
+    ]
 
     for idx, (hash_name, hash_value) in enumerate(stix_object["hashes".items()]):
         hash_name_binding_name = "hash_name" + str(idx)
@@ -84,21 +97,22 @@ def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_
 
 
 @add_method(HashesProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name)
 
+
 @add_method(IDProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(IntegerProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(ListProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     if isinstance(self.contained, ReferenceProperty):
         insert_statements = list()
 
@@ -106,7 +120,7 @@ def generate_insert_information(self, data_sink, name, stix_object, table_name,
         for idx, item in enumerate(stix_object[name]):
             bindings = {
                 "id": stix_object["id"],
-                "ref_id": item
+                "ref_id": item,
             }
             insert_statements.append(insert(table).values(bindings))
         return insert_statements
@@ -122,17 +136,17 @@ def generate_insert_information(self, data_sink, name, stix_object, table_name,
 
 
 @add_method(ReferenceProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(StringProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
 @add_method(TimestampProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):
+def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
     return {name: stix_object[name]}
 
 
@@ -149,7 +163,7 @@ def generate_insert_for_array_in_table(table, values, foreign_key_value):
     for idx, item in enumerate(values):
         bindings = {
             "id": foreign_key_value,
-            "ref_id": item
+            "ref_id": item,
         }
         insert_statements.append(insert(table).values(bindings))
 
@@ -169,7 +183,8 @@ def generate_insert_for_external_references(data_sink, stix_object):
 
         if "hashes" in er:
             insert_statements.extend(
-                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"))
+                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"),
+            )
 
     return insert_statements
 
@@ -259,14 +274,14 @@ def generate_insert_for_embedded_object(data_sink, stix_object, type_name, schem
     object_table = data_sink.tables_dictionary[table_name]
     sub_insert_statements = list()
     for name, prop in stix_object._properties.items():
-            if name in stix_object:
-                result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
-                if isinstance(result,dict):
-                    bindings.update(result)
-                elif isinstance(result,list):
-                    sub_insert_statements.extend(result)
-                else:
-                    raise(ValueError(result))
+        if name in stix_object:
+            result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
+            if isinstance(result, dict):
+                bindings.update(result)
+            elif isinstance(result, list):
+                sub_insert_statements.extend(result)
+            else:
+                raise ValueError("wrong type" + result)
 
     insert_statements.append(insert(object_table).values(bindings))
     insert_statements.extend(sub_insert_statements)
@@ -288,15 +303,14 @@ def generate_insert_for_object(data_sink, stix_object, schema_name):
 
     sub_insert_statements = list()
     for name, prop in stix_object._properties.items():
-        if name == 'id' or name not in core_properties:
-            if name in stix_object:
-                result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
-                if isinstance(result,dict):
-                    bindings.update(result)
-                elif isinstance(result,list):
-                    sub_insert_statements.extend(result)
-                else:
-                    raise(ValueError(result))
+        if (name == 'id' or name not in core_properties) and name in stix_object:
+            result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
+            if isinstance(result, dict):
+                bindings.update(result)
+            elif isinstance(result, list):
+                sub_insert_statements.extend(result)
+            else:
+                raise ValueError("wrong type" + result)
 
     insert_statements.append(insert(object_table).values(bindings))
     insert_statements.extend(sub_insert_statements)
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index d9efeba3..ced8453a 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -137,7 +137,7 @@ def create_external_references_tables(metadata):
             ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
             CheckConstraint(
                 "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-            )
+            ),
         ),
         Column("source_name", Text),
         Column("description", Text),
@@ -160,7 +160,7 @@ def create_core_table(metadata, schema_name):
             ),
             primary_key=True,
         ),
-        Column("spec_version", Text, default="2.1")
+        Column("spec_version", Text, default="2.1"),
     ]
     if schema_name == "sdo":
         sdo_columns = [
@@ -176,7 +176,7 @@ def create_core_table(metadata, schema_name):
             Column("revoked", Boolean),
             Column("confidence", Integer),
             Column("lang", Text),
-            Column("labels", ARRAY(Text))
+            Column("labels", ARRAY(Text)),
         ]
         columns.extend(sdo_columns)
     else:
@@ -544,7 +544,7 @@ def generate_object_table(
         table_name = table_name[0:30]
     if parent_table_name:
         table_name = parent_table_name + "_" + table_name
-    core_properties = SDO_COMMON_PROPERTIES if schema_name=="sdo" else SCO_COMMON_PROPERTIES
+    core_properties = SDO_COMMON_PROPERTIES if schema_name == "sdo" else SCO_COMMON_PROPERTIES
     columns = list()
     tables = list()
     for name, prop in properties.items():

From c93ddce7dfa8c01b625dfee6c80020f4c93c1985 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Fri, 29 Mar 2024 10:33:58 -0400
Subject: [PATCH 024/132] 3/29 checkpoint

---
 stix2/properties.py               | 56 +++++++++++++++++++++----------
 stix2/test/v21/test_properties.py | 45 ++++++++++++++-----------
 2 files changed, 63 insertions(+), 38 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 7b81bb88..c111103b 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -390,8 +390,17 @@ class DictionaryProperty(Property):
     def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
+        simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
         if not valid_types:
             valid_types = [StringProperty]
+        else:
+            for type_ in valid_types:
+                if isinstance(type_, ListProperty):
+                    if type_.contained not in simple_type:
+                        raise ValueError("Dictionary Property does not support lists of type: ", type_.contained)
+                elif type_ not in simple_type:
+                    raise ValueError("Dictionary Property does not support this value's type: ", type_)
+
         # elif not isinstance(valid_types, ListProperty):
         #     valid_types = [valid_types]
         
@@ -412,7 +421,6 @@ def clean(self, value, allow_custom=False):
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
         
-        valid_types = self.specifics
         for k in dictified.keys():
             if self.spec_version == '2.0':
                 if len(k) < 3:
@@ -443,24 +451,36 @@ def clean(self, value, allow_custom=False):
             #     if not isinstance(dictified[k], StringProperty) or not isinstance(dictified[k], IntegerProperty):
             #         raise ValueError("The dictionary expects values of type str or int")
 
-            simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
-            clear = False
+            # simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
+            # clear = False
+            # for type in self.valid_types:
+            #     if type in simple_type:
+            #         try:
+            #             self.valid_types.clean(dict[k])
+            #         except ValueError:
+            #             continue
+            #         clear = True
+            #     elif isinstance(dictified[k], ListProperty()):
+            #         list_type = dictified[k].contained
+            #         if list_type in simple_type:
+            #             for x in dictified[k]:
+            #                 list_type.clean(x)
+            #         else:
+            #             raise ValueError("Dictionary Property does not support lists of type: ", list_type)
+            #     if not clear:
+            #         raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
+            
+            clean = False
             for type in self.valid_types:
-                if type in simple_type:
-                    try:
-                        self.valid_types.clean(dict[k])
-                    except ValueError:
-                        continue
-                    clear = True
-                elif isinstance(dictified[k], ListProperty()):
-                    list_type = dictified[k].contained
-                    if list_type in simple_type:
-                        for x in dictified[k]:
-                            list_type.clean(x)
-                    else:
-                        raise ValueError("Dictionary Property does not support lists of type: ", list_type)
-                if not clear:
-                    raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
+                type_instance = type()
+                try:
+                    type_instance.clean(dictified[k])
+                    clean = True
+                except ValueError:
+                    continue
+            
+            if not clean:
+                raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 4aeeb8bc..260fc7a5 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -23,43 +23,48 @@ def test_dictionary_property():
         p.clean({})
 
 def test_dictionary_property_values_str():
-    p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1', value={'x': '123'})
-    assert p.clean()
+    p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
+    result = p.clean({'x': '123'}, False)
+    assert result == ({'x': '123'}, False)
 
-    q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1', value={'x': 123})
+    q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean()
+        assert q.clean({'x': 123})
 
 def test_dictionary_property_values_int():
-    p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1', value={'x': 123})
-    assert p.clean()
+    p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
+    result = p.clean({'x': 123}, False)
+    assert result == ({'x': 123}, False)
 
-    q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1', value={'x': '123'})
+    q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean()
+        assert q.clean({'x': '123'})
 
 def test_dictionary_property_values_stringlist():
-    p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1', value={'x': ['abc', 'def']})
-    assert p.clean()
+    p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
+    result = p.clean({'x': ['abc', 'def']}, False)
+    assert result == ({'x': ['abc', 'def']}, False)
 
-    q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1', value={'x': '123'})
+    q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean()
+        assert q.clean({'x': '123'})
 
-    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': [123, 456]})
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert r.clean()
+        assert r.clean({'x': [123, 456]})
 
 def test_dictionary_property_values_list():
-    p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': 123})
-    assert p.clean()
+    p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    result = p.clean({'x': 123}, False)
+    assert result == ({'x': 123}, False)
 
-    q = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': '123'})
-    assert q.clean()
+    q = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    result = q.clean({'x': '123'}, False)
+    assert result == ({'x': '123'}, False)
 
-    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1', value={'x': ['abc', 'def']})
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert r.clean()
+        assert r.clean({'x': ['abc', 'def']})
 
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'

From 9a59735daaf96b31478463e76f5b7d01d8b0ec10 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Fri, 29 Mar 2024 14:28:23 -0400
Subject: [PATCH 025/132] edits to observables

---
 stix2/properties.py      |  1 +
 stix2/v21/observables.py | 14 +++++++-------
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index c111103b..8a83fdf2 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -476,6 +476,7 @@ def clean(self, value, allow_custom=False):
                 try:
                     type_instance.clean(dictified[k])
                     clean = True
+                    break
                 except ValueError:
                     continue
             
diff --git a/stix2/v21/observables.py b/stix2/v21/observables.py
index 0929fd32..6fa777ed 100644
--- a/stix2/v21/observables.py
+++ b/stix2/v21/observables.py
@@ -181,7 +181,7 @@ class EmailMessage(_Observable):
         ('message_id', StringProperty()),
         ('subject', StringProperty()),
         ('received_lines', ListProperty(StringProperty)),
-        ('additional_header_fields', DictionaryProperty(valid_types="string_list", spec_version='2.1')),
+        ('additional_header_fields', DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')),
         ('body', StringProperty()),
         ('body_multipart', ListProperty(EmbeddedObjectProperty(type=EmailMIMEComponent))),
         ('raw_email_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
@@ -245,7 +245,7 @@ class PDFExt(_Extension):
     _properties = OrderedDict([
         ('version', StringProperty()),
         ('is_optimized', BooleanProperty()),
-        ('document_info_dict', DictionaryProperty(valid_types="string", spec_version='2.1')),
+        ('document_info_dict', DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')),
         ('pdfid0', StringProperty()),
         ('pdfid1', StringProperty()),
     ])
@@ -261,7 +261,7 @@ class RasterImageExt(_Extension):
         ('image_height', IntegerProperty()),
         ('image_width', IntegerProperty()),
         ('bits_per_pixel', IntegerProperty()),
-        ('exif_tags', DictionaryProperty(valid_types=["string", "integer"], spec_version='2.1')),
+        ('exif_tags', DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')),
     ])
 
 
@@ -468,7 +468,7 @@ class HTTPRequestExt(_Extension):
         ('request_method', StringProperty(required=True)),
         ('request_value', StringProperty(required=True)),
         ('request_version', StringProperty()),
-        ('request_header', DictionaryProperty(valid_types="string_list", spec_version='2.1')),
+        ('request_header', DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')),
         ('message_body_length', IntegerProperty()),
         ('message_body_data_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
     ])
@@ -496,7 +496,7 @@ class SocketExt(_Extension):
         ('address_family', EnumProperty(NETWORK_SOCKET_ADDRESS_FAMILY, required=True)),
         ('is_blocking', BooleanProperty()),
         ('is_listening', BooleanProperty()),
-        ('options', DictionaryProperty(valid_types="integer", spec_version='2.1')),
+        ('options', DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')),
         ('socket_type', EnumProperty(NETWORK_SOCKET_TYPE)),
         ('socket_descriptor', IntegerProperty(min=0)),
         ('socket_handle', IntegerProperty()),
@@ -550,7 +550,7 @@ class NetworkTraffic(_Observable):
         ('dst_byte_count', IntegerProperty(min=0)),
         ('src_packets', IntegerProperty(min=0)),
         ('dst_packets', IntegerProperty(min=0)),
-        ('ipfix', DictionaryProperty(valid_types=["string", "integer"], spec_version='2.1')),
+        ('ipfix', DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')),
         ('src_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
         ('dst_payload_ref', ReferenceProperty(valid_types='artifact', spec_version='2.1')),
         ('encapsulates_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))),
@@ -634,7 +634,7 @@ class Process(_Observable):
         ('created_time', TimestampProperty()),
         ('cwd', StringProperty()),
         ('command_line', StringProperty()),
-        ('environment_variables', DictionaryProperty(valid_types="string", spec_version='2.1')),
+        ('environment_variables', DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')),
         ('opened_connection_refs', ListProperty(ReferenceProperty(valid_types='network-traffic', spec_version='2.1'))),
         ('creator_user_ref', ReferenceProperty(valid_types='user-account', spec_version='2.1')),
         ('image_ref', ReferenceProperty(valid_types='file', spec_version='2.1')),

From 704b6ae0d5821338fdc9a5a6ee9fd680028bdbb1 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sat, 30 Mar 2024 09:44:51 -0400
Subject: [PATCH 026/132] fix sco fk

---
 .../datastore/relational_db/table_creation.py | 38 ++++++++++++++++++-
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index ced8453a..56c65b77 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -276,8 +276,36 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 @add_method(IDProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    foreign_key_column = "common.core_sdo.id" if kwargs.get("schema_name") else "common.core_sco.id"
+    schema_name = kwargs.get('schema_name')
+    if schema_name == "sro":
+        # sro common properties are the same as sdo's
+        schema_name = "sdo"
     table_name = kwargs.get("table_name")
+    if schema_name == "common":
+        return Column(
+            name,
+            Text,
+            CheckConstraint(
+                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
+            ),
+            primary_key=True,
+            nullable=not (self.required),
+        )
+    else:
+        foreign_key_column = f"common.core_{schema_name}.id"
+        return Column(
+            name,
+            Text,
+            ForeignKey(foreign_key_column, ondelete="CASCADE"),
+            CheckConstraint(
+                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
+            ),
+            primary_key=True,
+            nullable=not (self.required),
+        )
+
     return Column(
         name,
         Text,
@@ -542,9 +570,15 @@ def generate_object_table(
     # avoid long table names
     if table_name.startswith("extension-definition"):
         table_name = table_name[0:30]
+        table_name = table_name.replace("extension-definition-", "ext_def")
     if parent_table_name:
         table_name = parent_table_name + "_" + table_name
-    core_properties = SDO_COMMON_PROPERTIES if schema_name == "sdo" else SCO_COMMON_PROPERTIES
+    if schema_name in ["sdo", "sro"]:
+        core_properties = SDO_COMMON_PROPERTIES
+    elif schema_name == "sco":
+        core_properties = SCO_COMMON_PROPERTIES
+    else:
+        core_properties = list()
     columns = list()
     tables = list()
     for name, prop in properties.items():

From bf840b19011a299b0b770627ef95cceb52ade726 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 31 Mar 2024 16:26:17 -0400
Subject: [PATCH 027/132] handled sequences

---
 .../datastore/relational_db/input_creation.py | 743 ++++++++++--------
 .../datastore/relational_db/relational_db.py  |  11 +-
 .../datastore/relational_db/table_creation.py | 353 ++++-----
 3 files changed, 604 insertions(+), 503 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index dc6ab583..280613d8 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -1,325 +1,418 @@
-
-from sqlalchemy import insert
-
-from stix2.datastore.relational_db.add_method import add_method
-from stix2.datastore.relational_db.utils import (
-    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
-)
-from stix2.properties import (
-    BinaryProperty, BooleanProperty, DictionaryProperty,
-    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
-    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
-    Property, ReferenceProperty, StringProperty, TimestampProperty,
-)
-
-
-@add_method(Property)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    pass
-
-
-@add_method(BinaryProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(BooleanProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(DictionaryProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    bindings = {"id": stix_object["id"]}
-    table = data_sink.tables_dictionary[
-        canonicalize_table_name(
-            table_name + "_" + name,
-            schema_name,
-        )
-    ]
-    for idx, (name, value) in enumerate(stix_object.items()):
-        name_binding = f"name{idx}"
-        if len(self.value_types) == 1:
-            value_binding = f"value{idx}"
-        elif isinstance(value, int):
-            value_binding = f"integer_value{idx}"
-        else:
-            value_binding = f"string_value{idx}"
-
-        bindings[name_binding] = name
-        bindings[value_binding] = value
-
-        return [insert(table).values(bindings)]
-
-
-@add_method(EmbeddedObjectProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return generate_insert_for_embedded_object(data_sink, stix_object[name], name, schema_name)
-
-
-@add_method(EnumProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(ExtensionsProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    pass
-
-
-@add_method(FloatProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(HexProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name):
-    bindings = {"id": stix_object["id"]}
-    table = data_sink.tables_dictionary[
-        canonicalize_table_name(
-            table_name + "_" + name,
-            schema_name,
-        )
-    ]
-
-    for idx, (hash_name, hash_value) in enumerate(stix_object["hashes".items()]):
-        hash_name_binding_name = "hash_name" + str(idx)
-        hash_value_binding_name = "hash_value" + str(idx)
-
-        bindings[hash_name_binding_name] = hash_name
-        bindings[hash_value_binding_name] = hash_value
-
-    return [insert(table).values(bindings)]
-
-
-@add_method(HashesProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name)
-
-
-@add_method(IDProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(IntegerProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(ListProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    if isinstance(self.contained, ReferenceProperty):
-        insert_statements = list()
-
-        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
-        for idx, item in enumerate(stix_object[name]):
-            bindings = {
-                "id": stix_object["id"],
-                "ref_id": item,
-            }
-            insert_statements.append(insert(table).values(bindings))
-        return insert_statements
-    elif isinstance(self.contained, EmbeddedObjectProperty):
-        insert_statements = list()
-        for value in stix_object[name]:
-            insert_statements.extend(generate_insert_for_embedded_object(data_sink, value, table_name + "_" + name, None))
-            table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, None)]
-            insert_statements.append(insert(table).values({"id": stix_object["id"], "ref_id": 1}))
-        return insert_statements
-    else:
-        return {name: stix_object[name]}
-
-
-@add_method(ReferenceProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(StringProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-@add_method(TimestampProperty)
-def generate_insert_information(self, data_sink, name, stix_object, table_name, schema_name):  # noqa: F811
-    return {name: stix_object[name]}
-
-
-def derive_column_name(prop):
-    contained_property = prop.contained
-    if isinstance(contained_property, ReferenceProperty):
-        return "ref_id"
-    elif isinstance(contained_property, StringProperty):
-        return "value"
-
-
-def generate_insert_for_array_in_table(table, values, foreign_key_value):
-    insert_statements = list()
-    for idx, item in enumerate(values):
-        bindings = {
-            "id": foreign_key_value,
-            "ref_id": item,
-        }
-        insert_statements.append(insert(table).values(bindings))
-
-    return insert_statements
-
-
-def generate_insert_for_external_references(data_sink, stix_object):
-    insert_statements = list()
-    object_table = data_sink.tables_dictionary["common.external_references"]
-    for er in stix_object["external_references"]:
-        bindings = {"id": stix_object["id"]}
-        for prop in ["source_name", "description", "url", "external_id"]:
-            if prop in er:
-                bindings[prop] = er[prop]
-        er_insert_statement = insert(object_table).values(bindings)
-        insert_statements.append(er_insert_statement)
-
-        if "hashes" in er:
-            insert_statements.extend(
-                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"),
-            )
-
-    return insert_statements
-
-
-def generate_insert_for_granular_markings(granular_markings_table, stix_object):
-    granular_markings = stix_object["granular_markings"]
-    bindings = {
-        "id": stix_object["id"],
-    }
-    for idx, granular_marking in enumerate(granular_markings):
-        lang_binding_name = f"lang{idx}"
-        marking_ref_binding_name = f"marking_ref{idx}"
-        selectors_binding_name = f"selectors{idx}"
-
-        bindings[lang_binding_name] = granular_marking.get("lang")
-        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
-        bindings[selectors_binding_name] = granular_marking.get("selectors")
-
-    return [insert(granular_markings_table).values(bindings)]
-
-
-# def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
-#     sql_bindings_tuples = list()
-#     for name, ex in extensions.items():
-#         sql_bindings_tuples.extend(
-#             generate_insert_for_subtype_extension(
-#                 name,
-#                 ex,
-#                 foreign_key_value,
-#                 type_name,
-#                 core_properties,
-#             ),
-#         )
-#     return sql_bindings_tuples
-
-
-def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
-    if schema_name == "sdo":
-        core_table = data_sink.tables_dictionary["common.core_sdo"]
-    else:
-        core_table = data_sink.tables_dictionary["common.core_sco"]
-    insert_statements = list()
-    core_bindings = {}
-
-    for prop_name, value in stix_object.items():
-
-        if prop_name in core_properties:
-            # stored in separate tables below, skip here
-            if prop_name not in {"object_marking_refs", "granular_markings", "external_references", "type"}:
-                core_bindings[prop_name] = value
-
-    core_insert_statement = insert(core_table).values(core_bindings)
-    insert_statements.append(core_insert_statement)
-
-    if "object_marking_refs" in stix_object:
-        if schema_name == "sdo":
-            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
-        else:
-            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
-        insert_statements.extend(
-            generate_insert_for_array_in_table(
-                object_markings_ref_table,
-                stix_object["object_marking_refs"],
-                stix_object["id"],
-            ),
-        )
-
-    # Granular markings
-    if "granular_markings" in stix_object:
-        if schema_name == "sdo":
-            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
-        else:
-            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
-        granular_input_statements = generate_insert_for_granular_markings(
-            granular_marking_table,
-            stix_object.granular_markings,
-        )
-        insert_statements.extend(granular_input_statements)
-
-    return insert_statements
-
-
-def generate_insert_for_embedded_object(data_sink, stix_object, type_name, schema_name):
-    insert_statements = list()
-    bindings = dict()
-    table_name = canonicalize_table_name(type_name, schema_name)
-    object_table = data_sink.tables_dictionary[table_name]
-    sub_insert_statements = list()
-    for name, prop in stix_object._properties.items():
-        if name in stix_object:
-            result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
-            if isinstance(result, dict):
-                bindings.update(result)
-            elif isinstance(result, list):
-                sub_insert_statements.extend(result)
-            else:
-                raise ValueError("wrong type" + result)
-
-    insert_statements.append(insert(object_table).values(bindings))
-    insert_statements.extend(sub_insert_statements)
-    return insert_statements
-
-
-def generate_insert_for_object(data_sink, stix_object, schema_name):
-    insert_statements = list()
-    bindings = dict()
-    stix_id = stix_object["id"]
-    if schema_name == "sco":
-        core_properties = SCO_COMMON_PROPERTIES
-    else:
-        core_properties = SDO_COMMON_PROPERTIES
-    type_name = stix_object["type"]
-    table_name = canonicalize_table_name(type_name, schema_name)
-    object_table = data_sink.tables_dictionary[table_name]
-    insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
-
-    sub_insert_statements = list()
-    for name, prop in stix_object._properties.items():
-        if (name == 'id' or name not in core_properties) and name in stix_object:
-            result = prop.generate_insert_information(data_sink, name, stix_object, table_name, schema_name)
-            if isinstance(result, dict):
-                bindings.update(result)
-            elif isinstance(result, list):
-                sub_insert_statements.extend(result)
-            else:
-                raise ValueError("wrong type" + result)
-
-    insert_statements.append(insert(object_table).values(bindings))
-    insert_statements.extend(sub_insert_statements)
-
-    if "external_references" in stix_object:
-        insert_statements.extend(generate_insert_for_external_references(data_sink, stix_object))
-
-    if "extensions" in stix_object:
-        for ex in stix_object["extensions"]:
-            insert_statements.extend(generate_insert_for_object(data_sink, ex, schema_name, stix_id))
-
-    return insert_statements
+
+from sqlalchemy import insert
+
+from stix2.datastore.relational_db.add_method import add_method
+from stix2.datastore.relational_db.utils import (
+    SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+)
+from stix2.properties import (
+    BinaryProperty, BooleanProperty, DictionaryProperty,
+    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    Property, ReferenceProperty, StringProperty, TimestampProperty,
+)
+
+
+@add_method(Property)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    pass
+
+
+@add_method(BinaryProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(BooleanProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(DictionaryProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    bindings = dict()
+    data_sink = kwargs.get("data_sink")
+    table_name = kwargs.get("table_name")
+    schema_name = kwargs.get("schema_name")
+    foreign_key_value = kwargs.get("foreign_key_value")
+    insert_statements = list()
+    if "id" in stix_object:
+        bindings["id"] = stix_object["id"]
+    elif foreign_key_value:
+        bindings["id"] = foreign_key_value
+    table = data_sink.tables_dictionary[
+        canonicalize_table_name(
+            table_name + "_" + name,
+            schema_name,
+        )
+    ]
+    for name, value in stix_object[name].items():
+
+        name_binding = "name"
+        if not hasattr(self, "value_types") or len(self.value_types) == 1:
+            value_binding = "value"
+        elif isinstance(value, int):
+            value_binding = "integer_value"
+        else:
+            value_binding = "string_value"
+
+        bindings[name_binding] = name
+        bindings[value_binding] = value
+
+        insert_statements.append(insert(table).values(bindings))
+
+    return insert_statements
+
+
+@add_method(EmbeddedObjectProperty)
+def generate_insert_information(self, name, stix_object, is_list=False, foreign_key_value=None, **kwargs):  # noqa: F811
+    data_sink = kwargs.get("data_sink")
+    schema_name = kwargs.get("schema_name")
+    level = kwargs.get("level")
+    return generate_insert_for_sub_object(
+        data_sink, stix_object[name], self.type.__name__, schema_name,
+        level=level+1 if is_list else level,
+        is_embedded_object=True,
+        parent_table_name=kwargs.get("parent_table_name"),
+        foreign_key_value=foreign_key_value,
+    )
+
+
+@add_method(EnumProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(ExtensionsProperty)
+def generate_insert_information(self, name, stix_object, data_sink=None, table_name=None, schema_name=None, parent_table_name=None, **kwargs):  # noqa: F811
+    input_statements = list()
+    for ex_name, ex in stix_object["extensions"].items():
+        bindings = {
+            "id": stix_object["id"],
+            "ext_table_name": canonicalize_table_name(ex_name, schema_name),
+        }
+        ex_table = data_sink.tables_dictionary[table_name + "_" + "extensions"]
+        input_statements.append(insert(ex_table).values(bindings))
+        input_statements.extend(
+            generate_insert_for_sub_object(
+                data_sink, ex, ex_name, schema_name, stix_object["id"],
+                parent_table_name=parent_table_name,
+                is_extension=True,
+            ),
+        )
+    return input_statements
+
+
+@add_method(FloatProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(HexProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    v = bytes(stix_object[name], 'utf-8')
+    return {name: v}
+
+
+def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name, foreign_key_value=None, **kwargs):
+    if kwargs.get("is_embedded_object"):
+        if not kwargs.get("is_list") or kwargs.get("level") == 0:
+            bindings = {"id": stix_object["id"]}
+            # querky case where a property of an object is a single embedded objects
+            table_name = kwargs.get("parent_table_name")
+    else:
+        bindings = {"id": stix_object["id"]}
+    table = data_sink.tables_dictionary[
+        canonicalize_table_name(
+            table_name + "_" + name,
+            schema_name,
+        )
+    ]
+
+    for idx, (hash_name, hash_value) in enumerate(stix_object["hashes".items()]):
+        hash_name_binding_name = "hash_name" + str(idx)
+        hash_value_binding_name = "hash_value" + str(idx)
+
+        bindings[hash_name_binding_name] = hash_name
+        bindings[hash_value_binding_name] = hash_value
+
+    return [insert(table).values(bindings)]
+
+
+@add_method(HashesProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    data_sink = kwargs.get("data_sink")
+    table_name = kwargs.get("table_name")
+    schema_name = kwargs.get("schema_name")
+    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name)
+
+
+@add_method(IDProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(IntegerProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(ListProperty)
+def generate_insert_information(self, name, stix_object, level=0, is_extension=False, foreign_key_value=None, **kwargs):  # noqa: F811
+    data_sink = kwargs.get("data_sink")
+    table_name = kwargs.get("table_name")
+    if isinstance(self.contained, ReferenceProperty):
+        insert_statements = list()
+
+        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
+        for idx, item in enumerate(stix_object[name]):
+            bindings = {
+                "id": stix_object["id"],
+                "ref_id": item,
+            }
+            insert_statements.append(insert(table).values(bindings))
+        return insert_statements
+    elif isinstance(self.contained, EmbeddedObjectProperty):
+        insert_statements = list()
+        for value in stix_object[name]:
+            with data_sink.database_connection.begin() as trans:
+                next_id = trans.execute(data_sink.sequence)
+            table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
+            bindings = {
+                "id": foreign_key_value,
+                "ref_id": next_id,
+            }
+            insert_statements.append(insert(table).values(bindings))
+            insert_statements.extend(
+                generate_insert_for_sub_object(
+                    data_sink,
+                    value,
+                    table_name + "_" + name + "_" + self.contained.type.__name__,
+                    None,
+                    next_id,
+                    level,
+                    True,
+                    is_extension=is_extension,
+                ),
+            )
+        return insert_statements
+    else:
+        return {name: stix_object[name]}
+
+
+@add_method(ReferenceProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(StringProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+@add_method(TimestampProperty)
+def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+    return {name: stix_object[name]}
+
+
+def derive_column_name(prop):
+    contained_property = prop.contained
+    if isinstance(contained_property, ReferenceProperty):
+        return "ref_id"
+    elif isinstance(contained_property, StringProperty):
+        return "value"
+
+
+def generate_insert_for_array_in_table(table, values, foreign_key_value):
+    insert_statements = list()
+    for idx, item in enumerate(values):
+        bindings = {
+            "id": foreign_key_value,
+            "ref_id": item,
+        }
+        insert_statements.append(insert(table).values(bindings))
+    return insert_statements
+
+
+def generate_insert_for_external_references(data_sink, stix_object):
+    insert_statements = list()
+    object_table = data_sink.tables_dictionary["common.external_references"]
+    for er in stix_object["external_references"]:
+        bindings = {"id": stix_object["id"]}
+        for prop in ["source_name", "description", "url", "external_id"]:
+            if prop in er:
+                bindings[prop] = er[prop]
+        er_insert_statement = insert(object_table).values(bindings)
+        insert_statements.append(er_insert_statement)
+
+        if "hashes" in er:
+            insert_statements.extend(
+                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"),
+            )
+
+    return insert_statements
+
+
+def generate_insert_for_granular_markings(granular_markings_table, stix_object):
+    granular_markings = stix_object["granular_markings"]
+    bindings = {
+        "id": stix_object["id"],
+    }
+    for idx, granular_marking in enumerate(granular_markings):
+        lang_binding_name = f"lang{idx}"
+        marking_ref_binding_name = f"marking_ref{idx}"
+        selectors_binding_name = f"selectors{idx}"
+
+        bindings[lang_binding_name] = granular_marking.get("lang")
+        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
+        bindings[selectors_binding_name] = granular_marking.get("selectors")
+
+    return [insert(granular_markings_table).values(bindings)]
+
+
+# def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
+#     sql_bindings_tuples = list()
+#     for name, ex in extensions.items():
+#         sql_bindings_tuples.extend(
+#             generate_insert_for_subtype_extension(
+#                 name,
+#                 ex,
+#                 foreign_key_value,
+#                 type_name,
+#                 core_properties,
+#             ),
+#         )
+#     return sql_bindings_tuples
+
+
+def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
+    if schema_name == "sdo":
+        core_table = data_sink.tables_dictionary["common.core_sdo"]
+    else:
+        core_table = data_sink.tables_dictionary["common.core_sco"]
+    insert_statements = list()
+    core_bindings = {}
+
+    for prop_name, value in stix_object.items():
+
+        if prop_name in core_properties:
+            # stored in separate tables below, skip here
+            if prop_name not in {"object_marking_refs", "granular_markings", "external_references", "type"}:
+                core_bindings[prop_name] = value
+
+    core_insert_statement = insert(core_table).values(core_bindings)
+    insert_statements.append(core_insert_statement)
+
+    if "object_marking_refs" in stix_object:
+        if schema_name == "sdo":
+            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
+        else:
+            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
+        insert_statements.extend(
+            generate_insert_for_array_in_table(
+                object_markings_ref_table,
+                stix_object["object_marking_refs"],
+                stix_object["id"],
+            ),
+        )
+
+    # Granular markings
+    if "granular_markings" in stix_object:
+        if schema_name == "sdo":
+            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
+        else:
+            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
+        granular_input_statements = generate_insert_for_granular_markings(
+            granular_marking_table,
+            stix_object.granular_markings,
+        )
+        insert_statements.extend(granular_input_statements)
+
+    return insert_statements
+
+
+def generate_insert_for_sub_object(
+    data_sink, stix_object, type_name, schema_name, foreign_key_value=None,
+    is_embedded_object=False, is_list=False, parent_table_name=None, level=0,
+    is_extension=False,
+):
+    insert_statements = list()
+    bindings = dict()
+    if "id" in stix_object:
+        bindings["id"] = stix_object["id"]
+    elif foreign_key_value:
+        bindings["id"] = foreign_key_value
+    if parent_table_name and not is_extension:
+        type_name = parent_table_name + "_" + type_name
+    table_name = canonicalize_table_name(type_name, schema_name)
+    object_table = data_sink.tables_dictionary[table_name]
+    sub_insert_statements = list()
+    for name, prop in stix_object._properties.items():
+        if name in stix_object:
+            result = prop.generate_insert_information(
+                name,
+                stix_object,
+                data_sink=data_sink,
+                table_name=table_name,
+                schema_name=None,
+                foreign_key_value=foreign_key_value,
+                is_embedded_object=is_embedded_object,
+                is_list=is_list,
+                level=level,
+                is_extension=is_extension,
+                parent_table_name=table_name,
+            )
+            if isinstance(result, dict):
+                bindings.update(result)
+            elif isinstance(result, list):
+                sub_insert_statements.extend(result)
+            else:
+                raise ValueError("wrong type" + result)
+    if foreign_key_value:
+        bindings["id"] = foreign_key_value
+    insert_statements.append(insert(object_table).values(bindings))
+    insert_statements.extend(sub_insert_statements)
+    return insert_statements
+
+
+def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
+    insert_statements = list()
+    bindings = dict()
+    if schema_name == "sco":
+        core_properties = SCO_COMMON_PROPERTIES
+    else:
+        core_properties = SDO_COMMON_PROPERTIES
+    type_name = stix_object["type"]
+    table_name = canonicalize_table_name(type_name, schema_name)
+    object_table = data_sink.tables_dictionary[table_name]
+    insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
+    if "id" in stix_object:
+        foreign_key_value = stix_object["id"]
+    else:
+        foreign_key_value = None
+    sub_insert_statements = list()
+    for name, prop in stix_object._properties.items():
+        if (name == 'id' or name not in core_properties) and name in stix_object:
+            result = prop.generate_insert_information(
+                name, stix_object,
+                data_sink=data_sink,
+                table_name=table_name,
+                schema_name=schema_name,
+                parent_table_name=type_name,
+                level=level,
+                foreign_key_value=foreign_key_value,
+            )
+            if isinstance(result, dict):
+                bindings.update(result)
+            elif isinstance(result, list):
+                sub_insert_statements.extend(result)
+            else:
+                raise ValueError("wrong type" + result)
+
+    insert_statements.append(insert(object_table).values(bindings))
+    insert_statements.extend(sub_insert_statements)
+
+    if "external_references" in stix_object:
+        insert_statements.extend(generate_insert_for_external_references(data_sink, stix_object))
+
+    return insert_statements
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index e27562af..31e0ec75 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,5 +1,5 @@
 from sqlalchemy import MetaData, create_engine
-from sqlalchemy.schema import CreateSchema, CreateTable
+from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
@@ -149,6 +149,9 @@ def _create_schemas(self):
             trans.execute(CreateSchema("sro", if_not_exists=True))
 
     def _create_table_objects(self):
+        self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
+        with self.database_connection.begin() as trans:
+            print(trans.execute(self.sequence))
         tables = create_core_tables(self.metadata)
         for stix_class in _get_all_subclasses(_DomainObject):
             new_tables = generate_object_table(stix_class, self.metadata, "sdo")
@@ -170,11 +173,13 @@ def _create_table_objects(self):
         return tables
 
     def _instantiate_database(self):
-        self.metadata.create_all(self.database_connection.engine)
+        # self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
+        self.metadata.create_all(self.database_connection)
 
     def generate_stix_schema(self):
         for t in self.tables:
-            print(CreateTable(t).compile(self.database_connection.engine))
+            print(CreateTable(t).compile(self.database_connection))
+            print()
 
     def add(self, stix_data, version=None):
         _add(self, stix_data)
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 56c65b77..96743c9a 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -194,9 +194,9 @@ def determine_sql_type(self):
     return None
 
 
-@add_method(Property)
-def generate_table_information(self, name, **kwargs):
-    pass
+@add_method(BooleanProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Boolean
 
 
 @add_method(Property)
@@ -204,29 +204,9 @@ def determine_sql_type(self):  # noqa: F811
     pass
 
 
-@add_method(StringProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Text,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
-
-
-@add_method(StringProperty)
+@add_method(FloatProperty)
 def determine_sql_type(self):  # noqa: F811
-    return Text
-
-
-@add_method(IntegerProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Integer,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
+    return Float
 
 
 @add_method(IntegerProperty)
@@ -234,115 +214,34 @@ def determine_sql_type(self):  # noqa: F811
     return Integer
 
 
-@add_method(FloatProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Float,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
+@add_method(StringProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Text
 
+# ----------------------------- generate_table_information methods ----------------------------
 
-@add_method(FloatProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Float
 
+@add_method(Property)
+def generate_table_information(self, name, **kwargs):
+    pass
 
-@add_method(BooleanProperty)
+
+@add_method(BinaryProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        Boolean,
-        nullable=not self.required,
-        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
-    )
+    print("BinaryProperty not handled, yet")
+    return None
 
 
 @add_method(BooleanProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Boolean
-
-
-@add_method(TypeProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
     return Column(
         name,
-        Text,
+        Boolean,
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
-@add_method(IDProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    schema_name = kwargs.get('schema_name')
-    if schema_name == "sro":
-        # sro common properties are the same as sdo's
-        schema_name = "sdo"
-    table_name = kwargs.get("table_name")
-    if schema_name == "common":
-        return Column(
-            name,
-            Text,
-            CheckConstraint(
-                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-            nullable=not (self.required),
-        )
-    else:
-        foreign_key_column = f"common.core_{schema_name}.id"
-        return Column(
-            name,
-            Text,
-            ForeignKey(foreign_key_column, ondelete="CASCADE"),
-            CheckConstraint(
-                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-            nullable=not (self.required),
-        )
-
-    return Column(
-        name,
-        Text,
-        ForeignKey(foreign_key_column, ondelete="CASCADE"),
-        CheckConstraint(
-            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-        ),
-        primary_key=True,
-        nullable=not (self.required),
-    )
-
-
-@add_method(EnumProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    enum_re = "|".join(self.allowed)
-    return Column(
-        name,
-        Text,
-        CheckConstraint(
-            f"{name} ~ '^{enum_re}$'",
-        ),
-        nullable=not (self.required),
-    )
-
-
-@add_method(TimestampProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return Column(
-        name,
-        TIMESTAMP(timezone=True),
-        # CheckConstraint(
-        #     f"{name} ~ '^{enum_re}$'"
-        # ),
-        nullable=not (self.required),
-    )
-
-
 @add_method(DictionaryProperty)
 def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     columns = list()
@@ -394,6 +293,59 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
     return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
 
 
+@add_method(EmbeddedObjectProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
+    level = kwargs.get("level")
+    return generate_object_table(
+        self.type, metadata, schema_name, table_name, is_extension, True, is_list,
+        parent_table_name=table_name, level=level+1 if is_list else level,
+    )
+
+
+@add_method(EnumProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    enum_re = "|".join(self.allowed)
+    return Column(
+        name,
+        Text,
+        CheckConstraint(
+            f"{name} ~ '^{enum_re}$'",
+        ),
+        nullable=not (self.required),
+    )
+
+
+@add_method(ExtensionsProperty)
+def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
+    columns = list()
+    columns.append(
+        Column(
+            "id",
+            Text,
+            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "ext_table_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
+
+
+@add_method(FloatProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Float,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
 @add_method(HashesProperty)
 def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     level = kwargs.get("level")
@@ -428,75 +380,58 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     )
 
 
-@add_method(BinaryProperty)
+@add_method(IDProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    print("BinaryProperty not handled, yet")
-    return None
-
-
-@add_method(ExtensionsProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
-    columns = list()
-    columns.append(
-        Column(
-            "id",
-            Text,
-            ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
-            nullable=False,
-        ),
-    )
-    columns.append(
-        Column(
-            "ext_table_name",
+    schema_name = kwargs.get('schema_name')
+    if schema_name == "sro":
+        # sro common properties are the same as sdo's
+        schema_name = "sdo"
+    table_name = kwargs.get("table_name")
+    if schema_name == "common":
+        return Column(
+            name,
             Text,
-            nullable=False,
-        ),
-    )
-    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
-
-
-def ref_column(name, specifics, auth_type=0):
-    if specifics:
-        types = "|".join(specifics)
-        if auth_type == 0:
-            constraint = \
-                CheckConstraint(
-                    f"{name} ~ '^({types})" +
-                    "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                )
-        else:
-            constraint = \
-                CheckConstraint(
-                    f"(NOT({name} ~ '^({types})')) AND ({name} ~ " +
-                    "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
-                )
-        return Column(name, Text, constraint)
+            CheckConstraint(
+                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
+            ),
+            primary_key=True,
+            nullable=not (self.required),
+        )
     else:
+        foreign_key_column = f"common.core_{schema_name}.id"
         return Column(
             name,
             Text,
-            nullable=False,
+            ForeignKey(foreign_key_column, ondelete="CASCADE"),
+            CheckConstraint(
+                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
+            ),
+            primary_key=True,
+            nullable=not (self.required),
         )
 
-
-@add_method(ReferenceProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return ref_column(name, self.specifics, self.auth_type)
-
-
-@add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
-    level = kwargs.get("level")
-    return generate_object_table(
-        self.type, metadata, schema_name, table_name, is_extension, True, is_list,
-        parent_table_name=table_name, level=level+1 if is_list else level,
+    return Column(
+        name,
+        Text,
+        ForeignKey(foreign_key_column, ondelete="CASCADE"),
+        CheckConstraint(
+            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+        ),
+        primary_key=True,
+        nullable=not (self.required),
     )
 
 
-@add_method(ObjectReferenceProperty)
+@add_method(IntegerProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    table_name = kwargs.get('table_name')
-    raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
+    return Column(
+        name,
+        Integer,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
 
 
 @add_method(ListProperty)
@@ -558,6 +493,73 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 )
 
 
+def ref_column(name, specifics, auth_type=0):
+    if specifics:
+        types = "|".join(specifics)
+        if auth_type == 0:
+            constraint = \
+                CheckConstraint(
+                    f"{name} ~ '^({types})" +
+                    "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                )
+        else:
+            constraint = \
+                CheckConstraint(
+                    f"(NOT({name} ~ '^({types})')) AND ({name} ~ " +
+                    "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
+                )
+        return Column(name, Text, constraint)
+    else:
+        return Column(
+            name,
+            Text,
+            nullable=False,
+        )
+
+
+@add_method(ObjectReferenceProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    table_name = kwargs.get('table_name')
+    raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
+
+
+@add_method(ReferenceProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return ref_column(name, self.specifics, self.auth_type)
+
+
+@add_method(StringProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
+@add_method(TimestampProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        TIMESTAMP(timezone=True),
+        # CheckConstraint(
+        #     f"{name} ~ '^{enum_re}$'"
+        # ),
+        nullable=not (self.required),
+    )
+
+
+@add_method(TypeProperty)
+def generate_table_information(self, name, **kwargs):  # noqa: F811
+    return Column(
+        name,
+        Text,
+        nullable=not self.required,
+        default=self._fixed_value if hasattr(self, "_fixed_value") else None,
+    )
+
+
 def generate_object_table(
     stix_object_class, metadata, schema_name, foreign_key_name=None,
     is_extension=False, is_embedded_object=False, is_list=False, parent_table_name=None, level=0,
@@ -636,6 +638,7 @@ def generate_object_table(
                     ondelete="CASCADE",
                 ),
                 primary_key=True,
+                nullable=False,
             )
         else:
             column = Column(

From 85a4321968038c8ab66db60746892d4ce645bf08 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Mon, 1 Apr 2024 11:16:57 -0400
Subject: [PATCH 028/132] catch-all implemented

---
 stix2/properties.py                  | 121 +++++++++++----------------
 stix2/test/v21/test_observed_data.py |   4 +-
 stix2/test/v21/test_properties.py    |  14 ++--
 3 files changed, 59 insertions(+), 80 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 8a83fdf2..afba6599 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -133,7 +133,7 @@ class Property(object):
 
     Subclasses can also define the following functions:
 
-    - ``def clean(self, value, allow_custom) -> (any, has_custom):``
+    - ``def clean(self, value, allow_custom, strict) -> (any, has_custom):``
         - Return a value that is valid for this property, and enforce and
           detect value customization.  If ``value`` is not valid for this
           property, you may attempt to transform it first.  If ``value`` is not
@@ -148,7 +148,9 @@ class Property(object):
           mean there actually are any).  The method must return an appropriate
           value for has_custom.  Customization may not be applicable/possible
           for a property.  In that case, allow_custom can be ignored, and
-          has_custom must be returned as False.
+          has_custom must be returned as False. strict is a True/False flag 
+          that is used in the dictionary property. if strict is True, 
+          properties like StringProperty will be lenient in their clean method.
 
     - ``def default(self):``
         - provide a default value for this property.
@@ -191,7 +193,7 @@ def __init__(self, required=False, fixed=None, default=None):
         if default:
             self.default = default
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         return value, False
 
 
@@ -237,7 +239,10 @@ def clean(self, value, allow_custom):
         has_custom = False
         if isinstance(self.contained, Property):
             for item in value:
-                valid, temp_custom = self.contained.clean(item, allow_custom)
+                try:
+                    valid, temp_custom = self.contained.clean(item, allow_custom, strict=True)
+                except TypeError:
+                    valid, temp_custom = self.contained.clean(item, allow_custom)
                 result.append(valid)
                 has_custom = has_custom or temp_custom
 
@@ -275,8 +280,11 @@ class StringProperty(Property):
     def __init__(self, **kwargs):
         super(StringProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         if not isinstance(value, str):
+            if strict is True:
+                raise ValueError("Must be a string.")
+
             value = str(value)
         return value, False
 
@@ -296,7 +304,7 @@ def __init__(self, type, spec_version=DEFAULT_VERSION):
         self.spec_version = spec_version
         super(IDProperty, self).__init__()
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         _validate_id(value, self.spec_version, self.required_prefix)
         return value, False
 
@@ -311,7 +319,10 @@ def __init__(self, min=None, max=None, **kwargs):
         self.max = max
         super(IntegerProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
+        if strict is True and not isinstance(value, int):
+            raise ValueError("must be an integer.")
+        
         try:
             value = int(value)
         except Exception:
@@ -335,7 +346,10 @@ def __init__(self, min=None, max=None, **kwargs):
         self.max = max
         super(FloatProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
+        if strict is True and not isinstance(value, float):
+            raise ValueError("must be a float.")
+        
         try:
             value = float(value)
         except Exception:
@@ -356,7 +370,7 @@ class BooleanProperty(Property):
     _trues = ['true', 't', '1', 1, True]
     _falses = ['false', 'f', '0', 0, False]
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
 
         if isinstance(value, str):
             value = value.lower()
@@ -379,7 +393,7 @@ def __init__(self, precision="any", precision_constraint="exact", **kwargs):
 
         super(TimestampProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         return parse_into_datetime(
             value, self.precision, self.precision_constraint,
         ), False
@@ -390,27 +404,21 @@ class DictionaryProperty(Property):
     def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
-        simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
+        simple_types = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
         if not valid_types:
-            valid_types = [StringProperty]
+            valid_types = [Property]
         else:
             for type_ in valid_types:
                 if isinstance(type_, ListProperty):
-                    if type_.contained not in simple_type:
-                        raise ValueError("Dictionary Property does not support lists of type: ", type_.contained)
-                elif type_ not in simple_type:
+                    found = False
+                    for simple_type in simple_types:
+                        if isinstance(type_.contained, simple_type):
+                            found = True
+                    if not found:
+                        raise ValueError("Dictionary Property does not support lists of type: ", type_.contained, type(type_.contained))
+                elif type_ not in simple_types:
                     raise ValueError("Dictionary Property does not support this value's type: ", type_)
 
-        # elif not isinstance(valid_types, ListProperty):
-        #     valid_types = [valid_types]
-        
-        # if 'string_list' in valid_types and len(valid_types) > 1:
-        #     raise ValueError("'string_list' cannot be combined with other types in a list.")
-
-        # for type_ in valid_types:
-        #     if type_ not in ("string", "integer", "string_list"):
-        #         raise ValueError("The value of a dictionary key cannot be ", type_)
-
         self.valid_types = valid_types
 
         super(DictionaryProperty, self).__init__(**kwargs)
@@ -420,7 +428,7 @@ def clean(self, value, allow_custom=False):
             dictified = _get_dict(value)
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
-        
+
         for k in dictified.keys():
             if self.spec_version == '2.0':
                 if len(k) < 3:
@@ -437,51 +445,22 @@ def clean(self, value, allow_custom=False):
                     "underscore (_)"
                 )
                 raise DictionaryKeyError(k, msg)
-            
-            # if "string" in valid_types:
-            #     if not isinstance(dictified[k], StringProperty):
-            #         raise ValueError("The dictionary expects values of type str")
-            # elif "integer" in valid_types:
-            #     if not isinstance(dictified[k], IntegerProperty):
-            #         raise ValueError("The dictionary expects values of type int")
-            # elif "string_list" in valid_types:
-            #     if not isinstance(dictified[k], ListProperty(StringProperty)):
-            #         raise ValueError("The dictionary expects values of type list[str]")
-            # else:
-            #     if not isinstance(dictified[k], StringProperty) or not isinstance(dictified[k], IntegerProperty):
-            #         raise ValueError("The dictionary expects values of type str or int")
-
-            # simple_type = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
-            # clear = False
-            # for type in self.valid_types:
-            #     if type in simple_type:
-            #         try:
-            #             self.valid_types.clean(dict[k])
-            #         except ValueError:
-            #             continue
-            #         clear = True
-            #     elif isinstance(dictified[k], ListProperty()):
-            #         list_type = dictified[k].contained
-            #         if list_type in simple_type:
-            #             for x in dictified[k]:
-            #                 list_type.clean(x)
-            #         else:
-            #             raise ValueError("Dictionary Property does not support lists of type: ", list_type)
-            #     if not clear:
-            #         raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
-            
+
             clean = False
-            for type in self.valid_types:
-                type_instance = type()
-                try:
-                    type_instance.clean(dictified[k])
+            for type_ in self.valid_types:
+                if isinstance(type_, ListProperty):
+                    type_.clean(value=dictified[k], allow_custom=False)
                     clean = True
-                    break
-                except ValueError:
-                    continue
-            
+                else:
+                    type_instance = type_()
+                    try:
+                        type_instance.clean(value=dictified[k], allow_custom=False, strict=True)
+                        clean = True
+                        break
+                    except ValueError:
+                        continue
             if not clean:
-                raise ValueError("Dictionary Property does not support this value's type: ", self.valid_types)
+                raise ValueError("Dictionary Property does not support this value's type: ", type(dictified[k]))
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
@@ -504,7 +483,7 @@ def __init__(self, spec_hash_names, spec_version=DEFAULT_VERSION, **kwargs):
             if alg:
                 self.__alg_to_spec_name[alg] = spec_hash_name
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom, strict=False):
         # ignore the has_custom return value here; there is no customization
         # of DictionaryProperties.
         clean_dict, _ = super().clean(value, allow_custom)
@@ -552,7 +531,7 @@ def clean(self, value, allow_custom):
 
 class BinaryProperty(Property):
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         try:
             base64.b64decode(value)
         except (binascii.Error, TypeError):
@@ -562,7 +541,7 @@ def clean(self, value, allow_custom=False):
 
 class HexProperty(Property):
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         if not re.match(r"^([a-fA-F0-9]{2})+$", value):
             raise ValueError("must contain an even number of hexadecimal characters")
         return value, False
diff --git a/stix2/test/v21/test_observed_data.py b/stix2/test/v21/test_observed_data.py
index d2ccec49..91baa69e 100644
--- a/stix2/test/v21/test_observed_data.py
+++ b/stix2/test/v21/test_observed_data.py
@@ -1176,14 +1176,14 @@ def test_incorrect_socket_options():
         )
     assert "Incorrect options key" == str(excinfo.value)
 
-    with pytest.raises(ValueError) as excinfo:
+    with pytest.raises(Exception) as excinfo:
         stix2.v21.SocketExt(
             is_listening=True,
             address_family="AF_INET",
             socket_type="SOCK_STREAM",
             options={"SO_RCVTIMEO": '100'},
         )
-    assert "Options value must be an integer" == str(excinfo.value)
+    assert "Dictionary Property does not support this value's type" in str(excinfo.value)
 
 
 def test_network_traffic_tcp_example():
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 260fc7a5..fcb73a3c 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -7,8 +7,8 @@
 )
 from stix2.properties import (
     DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty,
-    HashesProperty, IDProperty, ListProperty, ObservableProperty,
-    ReferenceProperty, STIXObjectProperty, StringProperty, IntegerProperty
+    HashesProperty, IDProperty, IntegerProperty, ListProperty,
+    ObservableProperty, ReferenceProperty, STIXObjectProperty, StringProperty,
 )
 from stix2.v21.common import MarkingProperty
 
@@ -20,7 +20,7 @@ def test_dictionary_property():
 
     assert p.clean({'spec_version': '2.1'})
     with pytest.raises(ValueError):
-        p.clean({})
+        p.clean({}, False)
 
 def test_dictionary_property_values_str():
     p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
@@ -29,7 +29,7 @@ def test_dictionary_property_values_str():
 
     q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean({'x': 123})
+        assert q.clean({'x': [123]}, False)
 
 def test_dictionary_property_values_int():
     p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
@@ -38,7 +38,7 @@ def test_dictionary_property_values_int():
 
     q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean({'x': '123'})
+        assert q.clean({'x': [123]}, False)
 
 def test_dictionary_property_values_stringlist():
     p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
@@ -47,7 +47,7 @@ def test_dictionary_property_values_stringlist():
 
     q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert q.clean({'x': '123'})
+        assert q.clean({'x': [123]})
 
     r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
@@ -64,7 +64,7 @@ def test_dictionary_property_values_list():
 
     r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
     with pytest.raises(ValueError):
-        assert r.clean({'x': ['abc', 'def']})
+        assert r.clean({'x': ['abc', 'def']}, False)
 
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'

From 16302a0534dbffbb3eba4031fc9705a42eec7581 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Mon, 1 Apr 2024 11:17:30 -0400
Subject: [PATCH 029/132] linter changes

---
 stix2/properties.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index afba6599..6c99e600 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -148,8 +148,8 @@ class Property(object):
           mean there actually are any).  The method must return an appropriate
           value for has_custom.  Customization may not be applicable/possible
           for a property.  In that case, allow_custom can be ignored, and
-          has_custom must be returned as False. strict is a True/False flag 
-          that is used in the dictionary property. if strict is True, 
+          has_custom must be returned as False. strict is a True/False flag
+          that is used in the dictionary property. if strict is True,
           properties like StringProperty will be lenient in their clean method.
 
     - ``def default(self):``
@@ -322,7 +322,7 @@ def __init__(self, min=None, max=None, **kwargs):
     def clean(self, value, allow_custom=False, strict=False):
         if strict is True and not isinstance(value, int):
             raise ValueError("must be an integer.")
-        
+
         try:
             value = int(value)
         except Exception:
@@ -349,7 +349,7 @@ def __init__(self, min=None, max=None, **kwargs):
     def clean(self, value, allow_custom=False, strict=False):
         if strict is True and not isinstance(value, float):
             raise ValueError("must be a float.")
-        
+
         try:
             value = float(value)
         except Exception:

From 9dc75446e904c4747fb62beb40e781b4fd8acfd3 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Mon, 1 Apr 2024 12:32:27 -0400
Subject: [PATCH 030/132] typo

---
 stix2/properties.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 6c99e600..1c7a5c87 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -150,7 +150,7 @@ class Property(object):
           for a property.  In that case, allow_custom can be ignored, and
           has_custom must be returned as False. strict is a True/False flag
           that is used in the dictionary property. if strict is True,
-          properties like StringProperty will be lenient in their clean method.
+          properties like StringProperty will not be lenient in their clean method.
 
     - ``def default(self):``
         - provide a default value for this property.

From dcc83e693d040c47ce8f2f4c3976c6f41e16b64a Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 1 Apr 2024 13:32:35 -0400
Subject: [PATCH 031/132] no need to initialize sequence every time

---
 stix2/datastore/relational_db/relational_db.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 31e0ec75..10f1c8c8 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -150,8 +150,6 @@ def _create_schemas(self):
 
     def _create_table_objects(self):
         self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
-        with self.database_connection.begin() as trans:
-            print(trans.execute(self.sequence))
         tables = create_core_tables(self.metadata)
         for stix_class in _get_all_subclasses(_DomainObject):
             new_tables = generate_object_table(stix_class, self.metadata, "sdo")

From cdd970763726294274a73240aeec9000b0a8665b Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Mon, 1 Apr 2024 13:02:35 -0400
Subject: [PATCH 032/132] small chages

---
 stix2/properties.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 1c7a5c87..da6c2f79 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -404,10 +404,12 @@ class DictionaryProperty(Property):
     def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
-        simple_types = [BinaryProperty, BooleanProperty, FloatProperty, HashesProperty, HexProperty, IDProperty, IntegerProperty, StringProperty, TimestampProperty]
+        simple_types = [BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty, TimestampProperty, ReferenceProperty]
         if not valid_types:
             valid_types = [Property]
         else:
+            if not isinstance(valid_types, list):
+                valid_types = [valid_types]
             for type_ in valid_types:
                 if isinstance(type_, ListProperty):
                     found = False

From 86861f1772ef1a5a2f5feb8300313d54b86b684e Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 1 Apr 2024 15:49:00 -0400
Subject: [PATCH 033/132] minor changes for hashes

---
 .../datastore/relational_db/input_creation.py | 20 +++++++------------
 .../datastore/relational_db/table_creation.py |  2 +-
 2 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 280613d8..04f6e687 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -114,14 +114,9 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
     return {name: v}
 
 
-def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name, foreign_key_value=None, **kwargs):
-    if kwargs.get("is_embedded_object"):
-        if not kwargs.get("is_list") or kwargs.get("level") == 0:
-            bindings = {"id": stix_object["id"]}
-            # querky case where a property of an object is a single embedded objects
-            table_name = kwargs.get("parent_table_name")
-    else:
-        bindings = {"id": stix_object["id"]}
+def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name, foreign_key_value=None,
+                               is_embedded_object=False, **kwargs):
+    bindings = {"id": foreign_key_value}
     table = data_sink.tables_dictionary[
         canonicalize_table_name(
             table_name + "_" + name,
@@ -140,11 +135,10 @@ def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_
 
 
 @add_method(HashesProperty)
-def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
-    data_sink = kwargs.get("data_sink")
-    table_name = kwargs.get("table_name")
-    schema_name = kwargs.get("schema_name")
-    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name)
+def generate_insert_information(self, name, stix_object, data_sink=None, table_name=None, schema_name=None,
+                                is_embedded_object=False, foreign_key_value=None, **kwargs):  # noqa: F811
+    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name,
+                                      is_embedded_object=is_embedded_object, foreign_key_value=foreign_key_value)
 
 
 @add_method(IDProperty)
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 96743c9a..9cf7ca7f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -638,7 +638,7 @@ def generate_object_table(
                     ondelete="CASCADE",
                 ),
                 primary_key=True,
-                nullable=False,
+                nullable = False,
             )
         else:
             column = Column(

From 9311c662007a4fdfb1a11c88498cb95ec89b81b3 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Mon, 1 Apr 2024 15:53:30 -0400
Subject: [PATCH 034/132] Adding ListProperty leniency

---
 stix2/properties.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index da6c2f79..ff500f0f 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -226,7 +226,7 @@ def __init__(self, contained, **kwargs):
 
         super(ListProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom, strict_flag=False):
         try:
             iter(value)
         except TypeError:
@@ -240,7 +240,7 @@ def clean(self, value, allow_custom):
         if isinstance(self.contained, Property):
             for item in value:
                 try:
-                    valid, temp_custom = self.contained.clean(item, allow_custom, strict=True)
+                    valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict_flag)
                 except TypeError:
                     valid, temp_custom = self.contained.clean(item, allow_custom)
                 result.append(valid)
@@ -451,7 +451,7 @@ def clean(self, value, allow_custom=False):
             clean = False
             for type_ in self.valid_types:
                 if isinstance(type_, ListProperty):
-                    type_.clean(value=dictified[k], allow_custom=False)
+                    type_.clean(value=dictified[k], allow_custom=False, strict_flag=True)
                     clean = True
                 else:
                     type_instance = type_()

From ef399fe756be09f94f8c3f4aaa48e0ea5a6555ab Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 1 Apr 2024 17:34:30 -0400
Subject: [PATCH 035/132] hash table working

---
 .../datastore/relational_db/input_creation.py | 49 ++++++++++---------
 .../datastore/relational_db/table_creation.py |  2 +-
 2 files changed, 26 insertions(+), 25 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 04f6e687..40abdb80 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -65,7 +65,7 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(EmbeddedObjectProperty)
-def generate_insert_information(self, name, stix_object, is_list=False, foreign_key_value=None, **kwargs):  # noqa: F811
+def generate_insert_information(self, name, stix_object, is_list=False, foreign_key_value=None, is_extension=False, **kwargs):  # noqa: F811
     data_sink = kwargs.get("data_sink")
     schema_name = kwargs.get("schema_name")
     level = kwargs.get("level")
@@ -73,6 +73,7 @@ def generate_insert_information(self, name, stix_object, is_list=False, foreign_
         data_sink, stix_object[name], self.type.__name__, schema_name,
         level=level+1 if is_list else level,
         is_embedded_object=True,
+        is_extension=is_extension,
         parent_table_name=kwargs.get("parent_table_name"),
         foreign_key_value=foreign_key_value,
     )
@@ -114,31 +115,31 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
     return {name: v}
 
 
-def generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name, foreign_key_value=None,
-                               is_embedded_object=False, **kwargs):
+def generate_insert_for_hashes(
+    data_sink, name, stix_object, table_name, schema_name, foreign_key_value=None,
+    is_embedded_object=False, **kwargs,
+):
     bindings = {"id": foreign_key_value}
-    table = data_sink.tables_dictionary[
-        canonicalize_table_name(
-            table_name + "_" + name,
-            schema_name,
-        )
-    ]
-
-    for idx, (hash_name, hash_value) in enumerate(stix_object["hashes".items()]):
-        hash_name_binding_name = "hash_name" + str(idx)
-        hash_value_binding_name = "hash_value" + str(idx)
-
-        bindings[hash_name_binding_name] = hash_name
-        bindings[hash_value_binding_name] = hash_value
+    table_name = canonicalize_table_name(table_name + "_" + name, schema_name)
+    table = data_sink.tables_dictionary[table_name]
+    insert_statements = list()
+    for hash_name, hash_value in stix_object["hashes"].items():
 
-    return [insert(table).values(bindings)]
+        bindings["hash_name"] = hash_name
+        bindings["hash_value"] = hash_value
+        insert_statements.append(insert(table).values(bindings))
+    return insert_statements
 
 
 @add_method(HashesProperty)
-def generate_insert_information(self, name, stix_object, data_sink=None, table_name=None, schema_name=None,
-                                is_embedded_object=False, foreign_key_value=None, **kwargs):  # noqa: F811
-    return generate_insert_for_hashes(data_sink, name, stix_object, table_name, schema_name,
-                                      is_embedded_object=is_embedded_object, foreign_key_value=foreign_key_value)
+def generate_insert_information(   # noqa: F811
+    self, name, stix_object, data_sink=None, table_name=None, schema_name=None,
+    is_embedded_object=False, foreign_key_value=None, is_list=False, **kwargs,
+):
+    return generate_insert_for_hashes(
+        data_sink, name, stix_object, table_name, schema_name,
+        is_embedded_object=is_embedded_object, is_list=is_list, foreign_key_value=foreign_key_value,
+    )
 
 
 @add_method(IDProperty)
@@ -336,7 +337,7 @@ def generate_insert_for_sub_object(
         bindings["id"] = stix_object["id"]
     elif foreign_key_value:
         bindings["id"] = foreign_key_value
-    if parent_table_name and not is_extension:
+    if parent_table_name and (not is_extension or level > 0):
         type_name = parent_table_name + "_" + type_name
     table_name = canonicalize_table_name(type_name, schema_name)
     object_table = data_sink.tables_dictionary[table_name]
@@ -347,12 +348,12 @@ def generate_insert_for_sub_object(
                 name,
                 stix_object,
                 data_sink=data_sink,
-                table_name=table_name,
+                table_name=table_name if isinstance(prop, ListProperty) else parent_table_name,
                 schema_name=None,
                 foreign_key_value=foreign_key_value,
                 is_embedded_object=is_embedded_object,
                 is_list=is_list,
-                level=level,
+                level=level+1,
                 is_extension=is_extension,
                 parent_table_name=table_name,
             )
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 9cf7ca7f..96743c9a 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -638,7 +638,7 @@ def generate_object_table(
                     ondelete="CASCADE",
                 ),
                 primary_key=True,
-                nullable = False,
+                nullable=False,
             )
         else:
             column = Column(

From d5b7a14ef4faf1276df280aec42a463f7a69a326 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Sun, 31 Mar 2024 16:05:34 -0400
Subject: [PATCH 036/132] Experimental work on a relational db data source

---
 .../datastore/relational_db/relational_db.py  | 108 +++++++++++-------
 .../relational_db/relational_db_testing.py    |  26 ++++-
 .../datastore/relational_db/table_creation.py |  31 +++++
 stix2/datastore/relational_db/utils.py        |  74 ++++++++++++
 4 files changed, 192 insertions(+), 47 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 10f1c8c8..1b7a5ad1 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,4 +1,4 @@
-from sqlalchemy import MetaData, create_engine
+from sqlalchemy import MetaData, create_engine, select
 from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
 
 from stix2.base import _STIXBase
@@ -6,23 +6,13 @@
 from stix2.datastore.relational_db.input_creation import (
     generate_insert_for_object,
 )
-from stix2.datastore.relational_db.table_creation import (
-    create_core_tables, generate_object_table,
+from stix2.datastore.relational_db.table_creation import create_table_objects
+from stix2.datastore.relational_db.utils import (
+    canonicalize_table_name, schema_for, table_name_for,
 )
-from stix2.datastore.relational_db.utils import canonicalize_table_name
 from stix2.parsing import parse
-from stix2.v21.base import (
-    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
-)
-
-
-def _get_all_subclasses(cls):
-    all_subclasses = []
-
-    for subclass in cls.__subclasses__():
-        all_subclasses.append(subclass)
-        all_subclasses.extend(_get_all_subclasses(subclass))
-    return all_subclasses
+import stix2.registry
+import stix2.utils
 
 
 def _add(store, stix_data, allow_custom=True, version="2.1"):
@@ -124,21 +114,22 @@ class RelationalDBSink(DataSink):
     """
     def __init__(
         self, database_connection_url, allow_custom=True, version=None,
-        instantiate_database=True,
+        instantiate_database=True, *stix_object_classes
     ):
         super(RelationalDBSink, self).__init__()
         self.allow_custom = allow_custom
         self.metadata = MetaData()
         self.database_connection = create_engine(database_connection_url)
 
-        self._create_schemas()
-
-        self.tables = self._create_table_objects()
+        self.tables = create_table_objects(
+            self.metadata, stix_object_classes
+        )
         self.tables_dictionary = dict()
         for t in self.tables:
             self.tables_dictionary[canonicalize_table_name(t.name, t.schema)] = t
 
         if instantiate_database:
+            self._create_schemas()
             self._instantiate_database()
 
     def _create_schemas(self):
@@ -148,30 +139,8 @@ def _create_schemas(self):
             trans.execute(CreateSchema("sco", if_not_exists=True))
             trans.execute(CreateSchema("sro", if_not_exists=True))
 
-    def _create_table_objects(self):
-        self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
-        tables = create_core_tables(self.metadata)
-        for stix_class in _get_all_subclasses(_DomainObject):
-            new_tables = generate_object_table(stix_class, self.metadata, "sdo")
-            tables.extend(new_tables)
-        for stix_class in _get_all_subclasses(_RelationshipObject):
-            new_tables = generate_object_table(stix_class, self.metadata, "sro")
-            tables.extend(new_tables)
-        for stix_class in _get_all_subclasses(_Observable):
-            tables.extend(generate_object_table(stix_class, self.metadata, "sco"))
-        for stix_class in _get_all_subclasses(_MetaObject):
-            tables.extend(generate_object_table(stix_class, self.metadata, "common"))
-        for stix_class in _get_all_subclasses(_Extension):
-            if stix_class.extension_type not in ["new-sdo", "new-sco", "new-sro"]:
-                if hasattr(stix_class, "_applies_to"):
-                    schema_name = stix_class._applies_to
-                else:
-                    schema_name = "sco"
-                tables.extend(generate_object_table(stix_class, self.metadata, schema_name, is_extension=True))
-        return tables
-
     def _instantiate_database(self):
-        # self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
+        self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
         self.metadata.create_all(self.database_connection)
 
     def generate_stix_schema(self):
@@ -194,11 +163,62 @@ def insert_object(self, stix_object):
 
 
 class RelationalDBSource(DataSource):
+
+    def __init__(
+        self, database_connection_url, *stix_object_classes
+    ):
+        self.metadata = MetaData()
+        self.database_connection = create_engine(database_connection_url)
+        create_table_objects(
+            self.metadata, stix_object_classes
+        )
+
     def get(self, stix_id, version=None, _composite_filters=None):
-        pass
+
+        stix_type = stix2.utils.get_type_from_id(stix_id)
+        stix_class = stix2.registry.class_for_type(
+            # TODO: give user control over STIX version used?
+            stix_type, stix_version=stix2.DEFAULT_VERSION
+        )
+
+        # Info about the type-specific table
+        type_table_name = table_name_for(stix_type)
+        type_schema_name = schema_for(stix_class)
+        type_table = self.metadata.tables[f"{type_schema_name}.{type_table_name}"]
+
+        # Some fixed info about core tables
+        if type_schema_name == "sco":
+            core_table_name = "common.core_sco"
+        else:
+            # for SROs and SMOs too?
+            core_table_name = "common.core_sdo"
+
+        core_table = self.metadata.tables[core_table_name]
+
+        # Both core and type-specific tables have "id"; let's not duplicate
+        # that in the result set columns.  Is there a better way to do this?
+        type_cols_except_id = (
+            col for col in type_table.c if col.key != "id"
+        )
+
+        core_type_select = select(core_table, *type_cols_except_id) \
+            .join(type_table) \
+            .where(core_table.c.id == stix_id)
+
+        obj_dict = {}
+        with self.database_connection.begin() as conn:
+            # Should be at most one matching row
+            sco_data = conn.execute(core_type_select).mappings().first()
+            obj_dict.update(sco_data)
+
+        return stix_class(**obj_dict)
 
     def all_versions(self, stix_id, version=None, _composite_filters=None):
         pass
 
     def query(self, query=None):
         pass
+
+    def generate_stix_schema(self):
+        for t in self.metadata.tables.values():
+            print(CreateTable(t).compile(self.database_connection.engine))
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 695f735b..e4fb54ee 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -3,7 +3,10 @@
 import pytz
 
 import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBSink
+from stix2.datastore.relational_db.relational_db import (
+    RelationalDBSink, RelationalDBSource,
+)
+import stix2.properties
 
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
@@ -21,6 +24,7 @@
 )
 
 s = stix2.v21.Software(
+        id="software--28897173-7314-4eec-b1cf-2c625b635bf6",
         name="Word",
         cpe="cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
         swid="com.acme.rms-ce-v4-1-5-0",
@@ -94,8 +98,24 @@ def file_example_with_PDFExt_Object():
 
 
 def main():
-    store = RelationalDBSink("postgresql://localhost/stix-data-sink")
-    store.generate_stix_schema()
+    store = RelationalDBSink(
+        "postgresql://localhost/stix-data-sink",
+        False,
+        None,
+        True,
+        stix2.Directory
+    )
+    # store.generate_stix_schema()
+    store.add(directory_stix_object)
+
+    source = RelationalDBSource(
+        "postgresql://localhost/stix-data-sink",
+        stix2.Directory
+    )
+    source.generate_stix_schema()
+
+    read_obj = source.get(directory_stix_object.id)
+    print(read_obj)
 
 
 if __name__ == '__main__':
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 96743c9a..e8b31240 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -8,6 +8,7 @@
 from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
+    flat_classes, get_stix_object_classes, schema_for,
 )
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -16,6 +17,7 @@
     ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
     TimestampProperty, TypeProperty,
 )
+from stix2.v21.base import _Extension
 from stix2.v21.common import KillChainPhase
 
 
@@ -667,3 +669,32 @@ def create_core_tables(metadata):
     ]
     tables.extend(create_external_references_tables(metadata))
     return tables
+
+
+def create_table_objects(metadata, stix_object_classes):
+    if stix_object_classes:
+        # If classes are given, allow some flexibility regarding lists of
+        # classes vs single classes
+        stix_object_classes = flat_classes(stix_object_classes)
+
+    else:
+        # If no classes given explicitly, discover them automatically
+        stix_object_classes = get_stix_object_classes()
+
+    tables = create_core_tables(metadata)
+
+    for stix_class in stix_object_classes:
+
+        schema_name = schema_for(stix_class)
+        is_extension = issubclass(stix_class, _Extension)
+
+        tables.extend(
+            generate_object_table(
+                stix_class,
+                metadata,
+                schema_name,
+                is_extension=is_extension
+            )
+        )
+
+    return tables
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index e45897ac..42f16c03 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -1,4 +1,9 @@
+from collections.abc import Iterable, Mapping
 import inflection
+from stix2.v21.base import (
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
+)
+
 
 # Helps us know which data goes in core, and which in a type-specific table.
 SCO_COMMON_PROPERTIES = {
@@ -36,3 +41,72 @@ def canonicalize_table_name(table_name, schema_name=None):
         full_name = table_name
     full_name = full_name.replace("-", "_")
     return inflection.underscore(full_name)
+
+
+def _get_all_subclasses(cls):
+    all_subclasses = []
+
+    for subclass in cls.__subclasses__():
+        all_subclasses.append(subclass)
+        all_subclasses.extend(_get_all_subclasses(subclass))
+    return all_subclasses
+
+
+def get_stix_object_classes():
+    yield from _get_all_subclasses(_DomainObject)
+    yield from _get_all_subclasses(_RelationshipObject)
+    yield from _get_all_subclasses(_Observable)
+    yield from _get_all_subclasses(_MetaObject)
+    # Non-object extensions (property or toplevel-property only)
+    for ext_cls in _get_all_subclasses(_Extension):
+        if ext_cls.extension_type in (
+            "property-extension", "toplevel-property-extension"
+        ):
+            yield ext_cls
+
+
+def schema_for(stix_class):
+
+    if issubclass(stix_class, _DomainObject):
+        schema_name = "sdo"
+    elif issubclass(stix_class, _RelationshipObject):
+        schema_name = "sro"
+    elif issubclass(stix_class, _Observable):
+        schema_name = "sco"
+    elif issubclass(stix_class, _MetaObject):
+        schema_name = "common"
+    elif issubclass(stix_class, _Extension):
+        schema_name = getattr(stix_class, "_applies_to", "sco")
+    else:
+        schema_name = None
+
+    return schema_name
+
+
+def table_name_for(stix_type_or_class):
+    if isinstance(stix_type_or_class, str):
+        table_name = stix_type_or_class
+    else:
+        # A _STIXBase subclass
+        table_name = getattr(stix_type_or_class, "_type", stix_type_or_class.__name__)
+
+    # Applies to registered extension-definition style extensions only.
+    # Their "_type" attribute is actually set to the extension definition ID,
+    # rather than a STIX type.
+    if table_name.startswith("extension-definition"):
+        table_name = table_name[0:30]
+
+    table_name = canonicalize_table_name(table_name)
+    return table_name
+
+
+def flat_classes(class_or_classes):
+    if isinstance(class_or_classes, Iterable) and not isinstance(
+        # Try to generically detect STIX objects, which are iterable, but we
+        # don't want to iterate through those.
+        class_or_classes, Mapping
+    ):
+        for class_ in class_or_classes:
+            yield from flat_classes(class_)
+    else:
+        yield class_or_classes

From 32ccec076d8703de8068b185f07352c06a0ea285 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Mon, 1 Apr 2024 17:35:45 -0400
Subject: [PATCH 037/132] Change relational DB store to create table schemas
 and pass into source/sink so they aren't duplicating work.  Change the source
 sink constructor signatures to make this possible.

Also change docstrings to help readers understand the
constructor interfaces.
---
 .../datastore/relational_db/relational_db.py  | 189 +++++++++++-------
 .../relational_db/relational_db_testing.py    |  15 +-
 2 files changed, 120 insertions(+), 84 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 1b7a5ad1..2aa722cc 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -53,79 +53,97 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 
 
 class RelationalDBStore(DataStoreMixin):
-    """Interface to a file directory of STIX objects.
-
-    FileSystemStore is a wrapper around a paired FileSystemSink
-    and FileSystemSource.
-
-    Args:
-        stix_dir (str): path to directory of STIX objects
-        allow_custom (bool): whether to allow custom STIX content to be
-            pushed/retrieved. Defaults to True for FileSystemSource side
-            (retrieving data) and False for FileSystemSink
-            side(pushing data). However, when parameter is supplied, it
-            will be applied to both FileSystemSource and FileSystemSink.
-        bundlify (bool): whether to wrap objects in bundles when saving
-            them. Default: False.
-        encoding (str): The encoding to use when reading a file from the
-            filesystem.
-
-    Attributes:
-        source (FileSystemSource): FileSystemSource
-        sink (FileSystemSink): FileSystemSink
+    def __init__(
+        self, database_connection_url, allow_custom=True, version=None,
+        instantiate_database=True, *stix_object_classes
+    ):
+        """
+        Initialize this store.
+
+        Args:
+            database_connection_url: An SQLAlchemy URL referring to a database
+            allow_custom: Whether custom content is allowed when processing
+                dict content to be added to the store
+            version: TODO: unused so far
+            instantiate_database: Whether tables, etc should be created in the
+                database (only necessary the first time)
+            *stix_object_classes: STIX object classes to map into table schemas
+                (and ultimately database tables, if instantiation is desired).
+                This can be used to limit which table schemas are created, if
+                one is only working with a subset of STIX types.  If not given,
+                auto-detect all classes and create table schemas for all of
+                them.
+        """
+        database_connection = create_engine(database_connection_url)
 
-    """
-    def __init__(self, database_connection_url, allow_custom=None, encoding='utf-8'):
-        if allow_custom is None:
-            allow_custom_source = True
-            allow_custom_sink = False
-        else:
-            allow_custom_sink = allow_custom_source = allow_custom
+        self.metadata = MetaData()
+        create_table_objects(
+            self.metadata, stix_object_classes
+        )
 
-        super(RelationalDBStore, self).__init__(
-            source=RelationalDBSource(database_connection_url, allow_custom=allow_custom_source, encoding=encoding),
-            sink=RelationalDBSink(database_connection_url, allow_custom=allow_custom_sink),
+        super().__init__(
+            source=RelationalDBSource(
+                database_connection,
+                metadata=self.metadata
+            ),
+            sink=RelationalDBSink(
+                database_connection,
+                allow_custom=allow_custom,
+                version=version,
+                instantiate_database=instantiate_database,
+                metadata=self.metadata
+            ),
         )
 
 
 class RelationalDBSink(DataSink):
-    """Interface for adding/pushing STIX objects to an in-memory dictionary.
-
-    Designed to be paired with a MemorySource, together as the two
-    components of a MemoryStore.
-
-    Args:
-        stix_data (dict OR list): valid STIX 2.0 content in
-            bundle or a list.
-        _store (bool): whether the MemorySink is a part of a MemoryStore,
-            in which case "stix_data" is a direct reference to
-            shared memory with DataSource. Not user supplied
-        allow_custom (bool): whether to allow custom objects/properties
-            when exporting STIX content to file.
-            Default: True.
-        version (str): If present, it forces the parser to use the version
-            provided. Otherwise, the library will make the best effort based
-            on checking the "spec_version" property.
-
-    Attributes:
-        _data (dict): the in-memory dict that holds STIX objects.
-            If part of a MemoryStore, the dict is shared with a MemorySource
-
-    """
     def __init__(
-        self, database_connection_url, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes
+        self, database_connection_or_url, allow_custom=True, version=None,
+        instantiate_database=True, *stix_object_classes, metadata=None
     ):
+        """
+        Initialize this sink.  Only one of stix_object_classes and metadata
+        should be given: if the latter is given, assume table schemas are
+        already created.
+
+        Args:
+            database_connection_or_url: An SQLAlchemy engine object, or URL
+            allow_custom: Whether custom content is allowed when processing
+                dict content to be added to the sink
+            version: TODO: unused so far
+            instantiate_database: Whether tables, etc should be created in the
+                database (only necessary the first time)
+            *stix_object_classes: STIX object classes to map into table schemas
+                (and ultimately database tables, if instantiation is desired).
+                This can be used to limit which table schemas are created, if
+                one is only working with a subset of STIX types.  If not given,
+                auto-detect all classes and create table schemas for all of
+                them.  If metadata is given, the table data therein is used and
+                this argument is ignored.
+            metadata: SQLAlchemy MetaData object containing table information.
+                Only applicable when this class is instantiated via a store,
+                so that table information can be constructed once and shared
+                between source and sink.
+        """
         super(RelationalDBSink, self).__init__()
+
+        if isinstance(database_connection_or_url, str):
+            self.database_connection = create_engine(database_connection_or_url)
+        else:
+            self.database_connection = database_connection_or_url
+
+        if metadata:
+            self.metadata = metadata
+        else:
+            self.metadata = MetaData()
+            create_table_objects(
+                self.metadata, stix_object_classes
+            )
+
         self.allow_custom = allow_custom
-        self.metadata = MetaData()
-        self.database_connection = create_engine(database_connection_url)
 
-        self.tables = create_table_objects(
-            self.metadata, stix_object_classes
-        )
         self.tables_dictionary = dict()
-        for t in self.tables:
+        for t in self.metadata.tables.values():
             self.tables_dictionary[canonicalize_table_name(t.name, t.schema)] = t
 
         if instantiate_database:
@@ -144,7 +162,7 @@ def _instantiate_database(self):
         self.metadata.create_all(self.database_connection)
 
     def generate_stix_schema(self):
-        for t in self.tables:
+        for t in self.metadata.tables.values():
             print(CreateTable(t).compile(self.database_connection))
             print()
 
@@ -163,15 +181,42 @@ def insert_object(self, stix_object):
 
 
 class RelationalDBSource(DataSource):
-
     def __init__(
-        self, database_connection_url, *stix_object_classes
+        self, database_connection_or_url, *stix_object_classes, metadata=None
     ):
-        self.metadata = MetaData()
-        self.database_connection = create_engine(database_connection_url)
-        create_table_objects(
-            self.metadata, stix_object_classes
-        )
+        """
+        Initialize this source.  Only one of stix_object_classes and metadata
+        should be given: if the latter is given, assume table schemas are
+        already created.  Instances of this class do not create the actual
+        database tables; see the source/sink for that.
+
+        Args:
+            database_connection_or_url: An SQLAlchemy engine object, or URL
+            *stix_object_classes: STIX object classes to map into table schemas.
+                This can be used to limit which schemas are created, if one is
+                only working with a subset of STIX types.  If not given,
+                auto-detect all classes and create schemas for all of them.
+                If metadata is given, the table data therein is used and this
+                argument is ignored.
+            metadata: SQLAlchemy MetaData object containing table information.
+                Only applicable when this class is instantiated via a store,
+                so that table information can be constructed once and shared
+                between source and sink.
+        """
+        super().__init__()
+
+        if isinstance(database_connection_or_url, str):
+            self.database_connection = create_engine(database_connection_or_url)
+        else:
+            self.database_connection = database_connection_or_url
+
+        if metadata:
+            self.metadata = metadata
+        else:
+            self.metadata = MetaData()
+            create_table_objects(
+                self.metadata, stix_object_classes
+            )
 
     def get(self, stix_id, version=None, _composite_filters=None):
 
@@ -211,14 +256,10 @@ def get(self, stix_id, version=None, _composite_filters=None):
             sco_data = conn.execute(core_type_select).mappings().first()
             obj_dict.update(sco_data)
 
-        return stix_class(**obj_dict)
+        return stix_class(**obj_dict, allow_custom=True)
 
     def all_versions(self, stix_id, version=None, _composite_filters=None):
         pass
 
     def query(self, query=None):
         pass
-
-    def generate_stix_schema(self):
-        for t in self.metadata.tables.values():
-            print(CreateTable(t).compile(self.database_connection.engine))
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index e4fb54ee..7a8dd228 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -4,7 +4,7 @@
 
 import stix2
 from stix2.datastore.relational_db.relational_db import (
-    RelationalDBSink, RelationalDBSource,
+    RelationalDBSink, RelationalDBSource, RelationalDBStore
 )
 import stix2.properties
 
@@ -98,23 +98,18 @@ def file_example_with_PDFExt_Object():
 
 
 def main():
-    store = RelationalDBSink(
+    store = RelationalDBStore(
         "postgresql://localhost/stix-data-sink",
         False,
         None,
         True,
         stix2.Directory
     )
-    # store.generate_stix_schema()
-    store.add(directory_stix_object)
+    store.sink.generate_stix_schema()
 
-    source = RelationalDBSource(
-        "postgresql://localhost/stix-data-sink",
-        stix2.Directory
-    )
-    source.generate_stix_schema()
+    store.add(directory_stix_object)
 
-    read_obj = source.get(directory_stix_object.id)
+    read_obj = store.get(directory_stix_object.id)
     print(read_obj)
 
 

From b6c227615652bb5ce4016c9328603cde5e2eec50 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 10:02:24 -0400
Subject: [PATCH 038/132] flaky, put include all extensions

---
 .../datastore/relational_db/relational_db.py  | 19 +++++++++----------
 .../relational_db/relational_db_testing.py    |  6 ++----
 .../datastore/relational_db/table_creation.py |  4 ++--
 stix2/datastore/relational_db/utils.py        |  9 +++++----
 4 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 2aa722cc..0d634c33 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -55,7 +55,7 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 class RelationalDBStore(DataStoreMixin):
     def __init__(
         self, database_connection_url, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes
+        instantiate_database=True, *stix_object_classes,
     ):
         """
         Initialize this store.
@@ -78,20 +78,20 @@ def __init__(
 
         self.metadata = MetaData()
         create_table_objects(
-            self.metadata, stix_object_classes
+            self.metadata, stix_object_classes,
         )
 
         super().__init__(
             source=RelationalDBSource(
                 database_connection,
-                metadata=self.metadata
+                metadata=self.metadata,
             ),
             sink=RelationalDBSink(
                 database_connection,
                 allow_custom=allow_custom,
                 version=version,
                 instantiate_database=instantiate_database,
-                metadata=self.metadata
+                metadata=self.metadata,
             ),
         )
 
@@ -99,7 +99,7 @@ def __init__(
 class RelationalDBSink(DataSink):
     def __init__(
         self, database_connection_or_url, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes, metadata=None
+        instantiate_database=True, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this sink.  Only one of stix_object_classes and metadata
@@ -137,7 +137,7 @@ def __init__(
         else:
             self.metadata = MetaData()
             create_table_objects(
-                self.metadata, stix_object_classes
+                self.metadata, stix_object_classes,
             )
 
         self.allow_custom = allow_custom
@@ -164,7 +164,6 @@ def _instantiate_database(self):
     def generate_stix_schema(self):
         for t in self.metadata.tables.values():
             print(CreateTable(t).compile(self.database_connection))
-            print()
 
     def add(self, stix_data, version=None):
         _add(self, stix_data)
@@ -182,7 +181,7 @@ def insert_object(self, stix_object):
 
 class RelationalDBSource(DataSource):
     def __init__(
-        self, database_connection_or_url, *stix_object_classes, metadata=None
+        self, database_connection_or_url, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this source.  Only one of stix_object_classes and metadata
@@ -215,7 +214,7 @@ def __init__(
         else:
             self.metadata = MetaData()
             create_table_objects(
-                self.metadata, stix_object_classes
+                self.metadata, stix_object_classes,
             )
 
     def get(self, stix_id, version=None, _composite_filters=None):
@@ -223,7 +222,7 @@ def get(self, stix_id, version=None, _composite_filters=None):
         stix_type = stix2.utils.get_type_from_id(stix_id)
         stix_class = stix2.registry.class_for_type(
             # TODO: give user control over STIX version used?
-            stix_type, stix_version=stix2.DEFAULT_VERSION
+            stix_type, stix_version=stix2.DEFAULT_VERSION,
         )
 
         # Info about the type-specific table
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 7a8dd228..3e3cccce 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -3,9 +3,7 @@
 import pytz
 
 import stix2
-from stix2.datastore.relational_db.relational_db import (
-    RelationalDBSink, RelationalDBSource, RelationalDBStore
-)
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
 directory_stix_object = stix2.Directory(
@@ -103,7 +101,7 @@ def main():
         False,
         None,
         True,
-        stix2.Directory
+        stix2.Directory,
     )
     store.sink.generate_stix_schema()
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index e8b31240..36f91d1f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -693,8 +693,8 @@ def create_table_objects(metadata, stix_object_classes):
                 stix_class,
                 metadata,
                 schema_name,
-                is_extension=is_extension
-            )
+                is_extension=is_extension,
+            ),
         )
 
     return tables
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 42f16c03..eaef94b8 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -1,10 +1,11 @@
 from collections.abc import Iterable, Mapping
+
 import inflection
+
 from stix2.v21.base import (
     _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
 )
 
-
 # Helps us know which data goes in core, and which in a type-specific table.
 SCO_COMMON_PROPERTIES = {
     "id",
@@ -59,8 +60,8 @@ def get_stix_object_classes():
     yield from _get_all_subclasses(_MetaObject)
     # Non-object extensions (property or toplevel-property only)
     for ext_cls in _get_all_subclasses(_Extension):
-        if ext_cls.extension_type in (
-            "property-extension", "toplevel-property-extension"
+        if ext_cls.extension_type not in (
+            "new_sdo", "new_sco", "new_sro",
         ):
             yield ext_cls
 
@@ -104,7 +105,7 @@ def flat_classes(class_or_classes):
     if isinstance(class_or_classes, Iterable) and not isinstance(
         # Try to generically detect STIX objects, which are iterable, but we
         # don't want to iterate through those.
-        class_or_classes, Mapping
+        class_or_classes, Mapping,
     ):
         for class_ in class_or_classes:
             yield from flat_classes(class_)

From 6251c2826505f0f91f3f15bd8fe6e659ce863264 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 14:31:08 -0400
Subject: [PATCH 039/132] fix external_references, sequence, extension fix,
 again

---
 stix2/datastore/relational_db/input_creation.py | 10 +++++++++-
 stix2/datastore/relational_db/relational_db.py  |  2 +-
 stix2/datastore/relational_db/table_creation.py |  8 ++++++--
 stix2/datastore/relational_db/utils.py          |  3 ++-
 4 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 40abdb80..e6b6ee90 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -231,18 +231,26 @@ def generate_insert_for_array_in_table(table, values, foreign_key_value):
 
 def generate_insert_for_external_references(data_sink, stix_object):
     insert_statements = list()
+    next_id = None
     object_table = data_sink.tables_dictionary["common.external_references"]
     for er in stix_object["external_references"]:
         bindings = {"id": stix_object["id"]}
         for prop in ["source_name", "description", "url", "external_id"]:
             if prop in er:
                 bindings[prop] = er[prop]
+        if "hashes" in er:
+            with data_sink.database_connection.begin() as trans:
+                next_id = trans.execute(data_sink.sequence)
+            bindings["hash_ref_id"] = next_id
+        else:
+            # hash_ref_id is non-NULL, so -1 means there are no hashes
+            bindings["hash_ref_id"] = -1
         er_insert_statement = insert(object_table).values(bindings)
         insert_statements.append(er_insert_statement)
 
         if "hashes" in er:
             insert_statements.extend(
-                generate_insert_for_hashes(data_sink, "hashes", er["hashes"], "external_references_hashes", "sdo"),
+                generate_insert_for_hashes(data_sink, "hashes", er, "external_references", "common", foreign_key_value=next_id),
             )
 
     return insert_statements
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 0d634c33..db217eb4 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -139,6 +139,7 @@ def __init__(
             create_table_objects(
                 self.metadata, stix_object_classes,
             )
+        self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1, schema="common")
 
         self.allow_custom = allow_custom
 
@@ -158,7 +159,6 @@ def _create_schemas(self):
             trans.execute(CreateSchema("sro", if_not_exists=True))
 
     def _instantiate_database(self):
-        self.sequence = Sequence("my_general_seq", metadata=self.metadata, start=1)
         self.metadata.create_all(self.database_connection)
 
     def generate_stix_schema(self):
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 36f91d1f..9f37558f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -74,7 +74,7 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
             "id",
             key_type,
             ForeignKey(
-                canonicalize_table_name(table_name, schema_name) + ".id",
+                canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
                 ondelete="CASCADE",
             ),
 
@@ -145,10 +145,12 @@ def create_external_references_tables(metadata):
         Column("description", Text),
         Column("url", Text),
         Column("external_id", Text),
+        # all such keys are generated using the global sequence.
+        Column("hash_ref_id", Integer, primary_key=True, autoincrement = False)
     ]
     return [
         Table("external_references", metadata, *columns, schema="common"),
-        #  create_hashes_table("hashes", metadata, "common", "external_references")
+        create_hashes_table("hashes", metadata, "common", "external_references", Integer)
     ]
 
 
@@ -468,6 +470,8 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 Integer,
                 primary_key=True,
                 nullable=False,
+                # all such keys are generated using the global sequence.
+                autoincrement = False
             ),
         )
         tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index eaef94b8..bee8d54d 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -61,7 +61,7 @@ def get_stix_object_classes():
     # Non-object extensions (property or toplevel-property only)
     for ext_cls in _get_all_subclasses(_Extension):
         if ext_cls.extension_type not in (
-            "new_sdo", "new_sco", "new_sro",
+            "new-sdo", "new-sco", "new-sro",
         ):
             yield ext_cls
 
@@ -96,6 +96,7 @@ def table_name_for(stix_type_or_class):
     # rather than a STIX type.
     if table_name.startswith("extension-definition"):
         table_name = table_name[0:30]
+        table_name = table_name.replace("extension-definition-", "ext_def")
 
     table_name = canonicalize_table_name(table_name)
     return table_name

From e78a8374096c6dbcb0ea323e38219346014c90fb Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 14:36:03 -0400
Subject: [PATCH 040/132] flaky

---
 stix2/datastore/relational_db/table_creation.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 9f37558f..95ffc5c3 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -146,11 +146,11 @@ def create_external_references_tables(metadata):
         Column("url", Text),
         Column("external_id", Text),
         # all such keys are generated using the global sequence.
-        Column("hash_ref_id", Integer, primary_key=True, autoincrement = False)
+        Column("hash_ref_id", Integer, primary_key=True, autoincrement=False),
     ]
     return [
         Table("external_references", metadata, *columns, schema="common"),
-        create_hashes_table("hashes", metadata, "common", "external_references", Integer)
+        create_hashes_table("hashes", metadata, "common", "external_references", Integer),
     ]
 
 
@@ -471,7 +471,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 primary_key=True,
                 nullable=False,
                 # all such keys are generated using the global sequence.
-                autoincrement = False
+                autoincrement=False,
             ),
         )
         tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))

From 8a5685dae71ebaeae92eaf3a14b719f9e4d9faaa Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 15:16:54 -0400
Subject: [PATCH 041/132] handle list of enums

---
 stix2/datastore/relational_db/table_creation.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 95ffc5c3..1176a51f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -452,6 +452,21 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 schema_name,
             ),
         ]
+    elif isinstance(self.contained, EnumProperty):
+        columns = list()
+        columns.append(
+            Column(
+                "id",
+                Text,
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+                nullable=False,
+            ),
+        )
+        columns.append(self.contained.generate_table_information(name))
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
     elif isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(

From 4b6909535e6cdab8d12e98ee6ff1951d0ab15990 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 15:27:55 -0400
Subject: [PATCH 042/132] flaky

---
 stix2/test/v21/test_properties.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index fcb73a3c..0eff3c29 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -22,6 +22,7 @@ def test_dictionary_property():
     with pytest.raises(ValueError):
         p.clean({}, False)
 
+
 def test_dictionary_property_values_str():
     p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
     result = p.clean({'x': '123'}, False)
@@ -31,6 +32,7 @@ def test_dictionary_property_values_str():
     with pytest.raises(ValueError):
         assert q.clean({'x': [123]}, False)
 
+
 def test_dictionary_property_values_int():
     p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
     result = p.clean({'x': 123}, False)
@@ -40,6 +42,7 @@ def test_dictionary_property_values_int():
     with pytest.raises(ValueError):
         assert q.clean({'x': [123]}, False)
 
+
 def test_dictionary_property_values_stringlist():
     p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
     result = p.clean({'x': ['abc', 'def']}, False)
@@ -53,6 +56,7 @@ def test_dictionary_property_values_stringlist():
     with pytest.raises(ValueError):
         assert r.clean({'x': [123, 456]})
 
+
 def test_dictionary_property_values_list():
     p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
     result = p.clean({'x': 123}, False)
@@ -66,6 +70,7 @@ def test_dictionary_property_values_list():
     with pytest.raises(ValueError):
         assert r.clean({'x': ['abc', 'def']}, False)
 
+
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'
 

From 9321e061ca4954e6ea0b33882462db85d5ab4cde Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 2 Apr 2024 17:22:02 -0400
Subject: [PATCH 043/132] adding prelimiary test file for url obj

---
 stix2/test/test_url.py | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)
 create mode 100644 stix2/test/test_url.py

diff --git a/stix2/test/test_url.py b/stix2/test/test_url.py
new file mode 100644
index 00000000..99707a6c
--- /dev/null
+++ b/stix2/test/test_url.py
@@ -0,0 +1,25 @@
+import datetime as dt
+import pytest
+import stix2
+from stix2.datastore.realtional_db.relational_db import RelationalDBStore
+import stix2.properties
+
+url_stix_object = stix2.URL(
+    type = "url",
+    id = "url--c1477287-23ac-5971-a010-5c287877fa60",
+    value = "https://example.com/research/index.html"
+)
+
+store = RelationalDBStore(
+        "postgresql://localhost/stix-data-sink",
+        False,
+        None,
+        True,
+        stix2.URL,
+)
+
+def test_url():
+    store.sink.generate_stix_schema()
+    store.add(url_stix_object)
+    read_obj = store.get(url_stix_object)
+    assert read_obj == url_stix_object

From c0e83b897185a1589e71a3009418a77dd34caa68 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 2 Apr 2024 17:58:06 -0400
Subject: [PATCH 044/132] Dictionary Tables use valid_types

---
 stix2/datastore/relational_db/add_method.py   |  7 +-
 .../datastore/relational_db/table_creation.py | 65 ++++++++++++-------
 stix2/datastore/relational_db/utils.py        | 44 +++++++++++++
 stix2/properties.py                           |  5 +-
 4 files changed, 95 insertions(+), 26 deletions(-)

diff --git a/stix2/datastore/relational_db/add_method.py b/stix2/datastore/relational_db/add_method.py
index ad08a2e0..02d10c80 100644
--- a/stix2/datastore/relational_db/add_method.py
+++ b/stix2/datastore/relational_db/add_method.py
@@ -1,3 +1,5 @@
+import re
+
 # _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
 #
 #
@@ -7,9 +9,8 @@
 def create_real_method_name(name, klass_name):
     # if klass_name not in _ALLOWABLE_CLASSES:
     #     raise NameError
-    # split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
-    # split_up_klass_name.remove("Type")
-    return name + "_" + "_".join([x.lower() for x in klass_name])
+    split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
+    return name + "_" + "_".join([x.lower() for x in split_up_klass_name])
 
 
 def add_method(cls):
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 1176a51f..0b8b7446 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -8,7 +8,8 @@
 from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
-    flat_classes, get_stix_object_classes, schema_for,
+    determine_column_name, determine_sql_type_from_class, flat_classes,
+    get_stix_object_classes, schema_for,
 )
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -193,19 +194,24 @@ def create_core_table(metadata, schema_name):
     )
 
 
+@add_method(Property)
+def determine_sql_type(self):  # noqa: F811
+    pass
+
+
 @add_method(KillChainPhase)
-def determine_sql_type(self):
+def determine_sql_type(self):  # noqa: F811
     return None
 
 
-@add_method(BooleanProperty)
+@add_method(BinaryProperty)
 def determine_sql_type(self):  # noqa: F811
     return Boolean
 
 
-@add_method(Property)
+@add_method(BooleanProperty)
 def determine_sql_type(self):  # noqa: F811
-    pass
+    return Boolean
 
 
 @add_method(FloatProperty)
@@ -213,15 +219,31 @@ def determine_sql_type(self):  # noqa: F811
     return Float
 
 
+@add_method(HexProperty)
+def determine_sql_type(self):  # noqa: F811
+    return LargeBinary
+
+
 @add_method(IntegerProperty)
 def determine_sql_type(self):  # noqa: F811
     return Integer
 
 
+@add_method(ReferenceProperty)
+def determine_sql_type(self):  # noqa: F811
+    return Text
+
+
 @add_method(StringProperty)
 def determine_sql_type(self):  # noqa: F811
     return Text
 
+
+@add_method(TimestampProperty)
+def determine_sql_type(self):  # noqa: F811
+    return TIMESTAMP(timezone=True)
+
+
 # ----------------------------- generate_table_information methods ----------------------------
 
 
@@ -264,36 +286,35 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
             nullable=False,
         ),
     )
-    if len(self.specifics) == 1:
-        if self.specifics[0] != "string_list":
+    if len(self.valid_types) == 1:
+        if not isinstance(self.valid_types[0], ListProperty):
             columns.append(
                 Column(
                     "value",
-                    Text if self.specifics[0] == "string" else Integer,
+                    # its a class
+                    determine_sql_type_from_class(self.valid_types[0]),
                     nullable=False,
                 ),
             )
         else:
+            contained_class = self.valid_types[0].contained
             columns.append(
                 Column(
                     "value",
-                    ARRAY(Text),
+                    # its an instance, not a class
+                    ARRAY(contained_class.determine_sql_type()),
                     nullable=False,
                 ),
             )
     else:
-        columns.append(
-            Column(
-                "string_value",
-                Text,
-            ),
-        )
-        columns.append(
-            Column(
-                "integer_value",
-                Integer,
-            ),
-        )
+        for column_type in self.valid_types:
+            sql_type = determine_sql_type_from_class(column_type)
+            columns.append(
+                Column(
+                    determine_column_name(column_type),
+                    sql_type,
+                ),
+            )
     return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
 
 
@@ -315,7 +336,7 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
         CheckConstraint(
             f"{name} ~ '^{enum_re}$'",
         ),
-        nullable=not (self.required),
+        nullable=not self.required,
     )
 
 
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index bee8d54d..0958958f 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -1,7 +1,15 @@
 from collections.abc import Iterable, Mapping
 
 import inflection
+from sqlalchemy import (  # create_engine,; insert,
+    TIMESTAMP, Boolean, Float, Integer, LargeBinary, Text,
+)
 
+from stix2.properties import (
+    BinaryProperty, BooleanProperty, FloatProperty, HexProperty,
+    IntegerProperty, Property, ReferenceProperty, StringProperty,
+    TimestampProperty,
+)
 from stix2.v21.base import (
     _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
 )
@@ -112,3 +120,39 @@ def flat_classes(class_or_classes):
             yield from flat_classes(class_)
     else:
         yield class_or_classes
+
+
+def determine_sql_type_from_class(cls):  # noqa: F811
+    if cls == BinaryProperty:
+        return LargeBinary
+    elif cls == BooleanProperty:
+        return Boolean
+    elif cls == FloatProperty:
+        return Float
+    elif cls == HexProperty:
+        return LargeBinary
+    elif cls == IntegerProperty:
+        return Integer
+    elif cls == StringProperty or cls == ReferenceProperty:
+        return Text
+    elif cls == TimestampProperty:
+        return TIMESTAMP(timezone=True)
+    elif cls == Property:
+        return Text
+
+
+def determine_column_name(cls):  # noqa: F811
+    if cls == BinaryProperty:
+        return "binary_value"
+    elif cls == BooleanProperty:
+        return "boolean_value"
+    elif cls == FloatProperty:
+        return "float_value"
+    elif cls == HexProperty:
+        return "hex_value"
+    elif cls == IntegerProperty:
+        return "integer_value"
+    elif cls == StringProperty or cls == ReferenceProperty:
+        return "string_value"
+    elif cls == TimestampProperty:
+        return "timestamp_value"
diff --git a/stix2/properties.py b/stix2/properties.py
index a733c042..a535cd02 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -404,7 +404,10 @@ class DictionaryProperty(Property):
     def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
 
-        simple_types = [BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty, TimestampProperty, ReferenceProperty]
+        simple_types = [
+            BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty,
+            TimestampProperty, ReferenceProperty,
+        ]
         if not valid_types:
             valid_types = [Property]
         else:

From be63e1f1e3432deb8144fe9e29151dfdfea05dfd Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Wed, 3 Apr 2024 19:47:54 -0400
Subject: [PATCH 045/132] moving test_url to v21 dir and adding better compare

---
 stix2/test/test_url.py     | 25 -------------------------
 stix2/test/v21/test_url.py | 29 +++++++++++++++++++++++++++++
 2 files changed, 29 insertions(+), 25 deletions(-)
 delete mode 100644 stix2/test/test_url.py
 create mode 100644 stix2/test/v21/test_url.py

diff --git a/stix2/test/test_url.py b/stix2/test/test_url.py
deleted file mode 100644
index 99707a6c..00000000
--- a/stix2/test/test_url.py
+++ /dev/null
@@ -1,25 +0,0 @@
-import datetime as dt
-import pytest
-import stix2
-from stix2.datastore.realtional_db.relational_db import RelationalDBStore
-import stix2.properties
-
-url_stix_object = stix2.URL(
-    type = "url",
-    id = "url--c1477287-23ac-5971-a010-5c287877fa60",
-    value = "https://example.com/research/index.html"
-)
-
-store = RelationalDBStore(
-        "postgresql://localhost/stix-data-sink",
-        False,
-        None,
-        True,
-        stix2.URL,
-)
-
-def test_url():
-    store.sink.generate_stix_schema()
-    store.add(url_stix_object)
-    read_obj = store.get(url_stix_object)
-    assert read_obj == url_stix_object
diff --git a/stix2/test/v21/test_url.py b/stix2/test/v21/test_url.py
new file mode 100644
index 00000000..1832134a
--- /dev/null
+++ b/stix2/test/v21/test_url.py
@@ -0,0 +1,29 @@
+import datetime as dt
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+url_dict = {
+    "type": "url",
+    "id": "url--c1477287-23ac-5971-a010-5c287877fa60",
+    "value" : "https://example.com/research/index.html"
+}
+
+store = RelationalDBStore(
+        "postgresql://localhost/stix-data-sink",
+        False,
+        None,
+        True
+)
+
+def test_url():
+    store.sink.generate_stix_schema()
+    url_stix_object = stix2.parse(url_dict)
+    store.add(url_stix_object)
+    read_obj = json.loads(store.get(url_stix_object).serialize())
+
+    for attrib in url_dict.keys():
+        assert url_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From def9f19c4e2099dca9ce2ddee8019da956cfc4f9 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 3 Apr 2024 20:45:12 -0400
Subject: [PATCH 046/132] handle kill-chains, support binary properties,
 special case hash table fk

---
 .../datastore/relational_db/input_creation.py |   2 +-
 .../datastore/relational_db/table_creation.py | 101 +++++++++++++++---
 2 files changed, 87 insertions(+), 16 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index e6b6ee90..7913de3c 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -356,7 +356,7 @@ def generate_insert_for_sub_object(
                 name,
                 stix_object,
                 data_sink=data_sink,
-                table_name=table_name if isinstance(prop, ListProperty) else parent_table_name,
+                table_name=table_name if isinstance(prop, (DictionaryProperty, ListProperty)) else parent_table_name,
                 schema_name=None,
                 foreign_key_value=foreign_key_value,
                 is_embedded_object=is_embedded_object,
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 0b8b7446..0b6e605d 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -69,29 +69,75 @@ def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_n
 
 
 def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text, level=1):
+    columns = list()
+    # special case, perhaps because its a single embedded object with hashes, and not a list of embedded object
+    # making the parent table's primary key does seem to worl
+    if table_name == "windows-pebinary-ext_WindowsPEOptionalHeaderType":
+        columns.append(
+            Column(
+                "id",
+                key_type,
+                # ForeignKey(
+                #     canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
+                #     ondelete="CASCADE",
+                # ),
+
+                nullable=False,
+            ),
+        )
+    else:
+        columns.append(
+            Column(
+                "id",
+                key_type,
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
+                    ondelete="CASCADE",
+                ),
+
+                nullable=False,
+            ),
+        )
+    columns.append(
+        Column(
+            "hash_name",
+            Text,
+            nullable=False,
+        ),
+    )
+    columns.append(
+        Column(
+            "hash_value",
+            Text,
+            nullable=False,
+        ),
+    )
+    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
+
+
+def create_kill_chain_phases_table(name, metadata, schema_name, table_name):
     columns = list()
     columns.append(
         Column(
             "id",
-            key_type,
+            Text,
             ForeignKey(
-                canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
+                canonicalize_table_name(table_name, schema_name) + ".id",
                 ondelete="CASCADE",
             ),
-
             nullable=False,
         ),
     )
     columns.append(
         Column(
-            "hash_name",
+            "kill_chain_name",
             Text,
             nullable=False,
         ),
     )
     columns.append(
         Column(
-            "hash_value",
+            "phase_name",
             Text,
             nullable=False,
         ),
@@ -246,16 +292,34 @@ def determine_sql_type(self):  # noqa: F811
 
 # ----------------------------- generate_table_information methods ----------------------------
 
+@add_method(KillChainPhase)
+def generate_table_information(  # noqa: F811
+        self, name, metadata, schema_name, table_name, is_extension=False, is_list=False,
+        **kwargs,
+):
+    level = kwargs.get("level")
+    return generate_object_table(
+        self.type, metadata, schema_name, table_name, is_extension, True, is_list,
+        parent_table_name=table_name, level=level + 1 if is_list else level,
+    )
+
 
 @add_method(Property)
-def generate_table_information(self, name, **kwargs):
+def generate_table_information(self, name, **kwargs):   # noqa: F811
     pass
 
 
 @add_method(BinaryProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
-    print("BinaryProperty not handled, yet")
-    return None
+    return Column(
+        name,
+        Text,
+        CheckConstraint(
+            # this regular expression might accept or reject some legal base64 strings
+            f"{name} ~  " + "'^[-A-Za-z0-9+/]*={0,3}$'",
+        ),
+        nullable=not self.required,
+    )
 
 
 @add_method(BooleanProperty)
@@ -374,12 +438,9 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 @add_method(HashesProperty)
 def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     level = kwargs.get("level")
-    parent_table_name = kwargs.get("parent_table_name")
     if kwargs.get("is_embedded_object"):
         if not kwargs.get("is_list") or level == 0:
             key_type = Text
-            # querky case where a property of an object is a single embedded objects
-            table_name = parent_table_name
         else:
             key_type = Integer
     else:
@@ -524,6 +585,9 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
             ),
         )
         return tables
+    elif self.contained == KillChainPhase:
+        tables.append(create_kill_chain_phases_table("kill_chain_phase", metadata, schema_name, table_name))
+        return tables
     else:
         if isinstance(self.contained, Property):
             sql_type = self.contained.determine_sql_type()
@@ -617,7 +681,9 @@ def generate_object_table(
         table_name = table_name.replace("extension-definition-", "ext_def")
     if parent_table_name:
         table_name = parent_table_name + "_" + table_name
-    if schema_name in ["sdo", "sro"]:
+    if is_embedded_object:
+        core_properties = list()
+    elif schema_name in ["sdo", "sro"]:
         core_properties = SDO_COMMON_PROPERTIES
     elif schema_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES
@@ -670,6 +736,8 @@ def generate_object_table(
                         canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if is_list else ".id"),
                         ondelete="CASCADE",
                     ),
+                    # if it is a not list, then it is a single embedded object, and the primary key is unique
+                    # primary_key=not is_list
                 )
         elif level > 0 and is_embedded_object:
             column = Column(
@@ -693,9 +761,12 @@ def generate_object_table(
             )
         columns.append(column)
 
-    all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
-    all_tables.extend(tables)
-    return all_tables
+    # all_tables = [Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name)]
+    # all_tables.extend(tables)
+    # return all_tables
+
+    tables.append(Table(canonicalize_table_name(table_name), metadata, *columns, schema=schema_name))
+    return tables
 
 
 def create_core_tables(metadata):

From eb4b2f1474a3170ca0a8275f6e209548c50fa068 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 3 Apr 2024 21:55:23 -0400
Subject: [PATCH 047/132] ignore new extensions, use the correct table name for
 property extensions

---
 .../datastore/relational_db/input_creation.py | 34 ++++++++++++-------
 1 file changed, 21 insertions(+), 13 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 7913de3c..8039fedb 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -88,19 +88,24 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 def generate_insert_information(self, name, stix_object, data_sink=None, table_name=None, schema_name=None, parent_table_name=None, **kwargs):  # noqa: F811
     input_statements = list()
     for ex_name, ex in stix_object["extensions"].items():
-        bindings = {
-            "id": stix_object["id"],
-            "ext_table_name": canonicalize_table_name(ex_name, schema_name),
-        }
-        ex_table = data_sink.tables_dictionary[table_name + "_" + "extensions"]
-        input_statements.append(insert(ex_table).values(bindings))
-        input_statements.extend(
-            generate_insert_for_sub_object(
-                data_sink, ex, ex_name, schema_name, stix_object["id"],
-                parent_table_name=parent_table_name,
-                is_extension=True,
-            ),
-        )
+        # ignore new extensions - they have no properties
+        if ex.extension_type and not ex.extension_type.startswith("new"):
+            if ex_name.startswith("extension-definition"):
+                ex_name = ex_name[0:30]
+                ex_name = ex_name.replace("extension-definition-", "ext_def")
+            bindings = {
+                "id": stix_object["id"],
+                "ext_table_name": canonicalize_table_name(ex_name, schema_name),
+            }
+            ex_table = data_sink.tables_dictionary[table_name + "_" + "extensions"]
+            input_statements.append(insert(ex_table).values(bindings))
+            input_statements.extend(
+                generate_insert_for_sub_object(
+                    data_sink, ex, ex_name, schema_name, stix_object["id"],
+                    parent_table_name=parent_table_name,
+                    is_extension=True,
+                ),
+            )
     return input_statements
 
 
@@ -347,6 +352,9 @@ def generate_insert_for_sub_object(
         bindings["id"] = foreign_key_value
     if parent_table_name and (not is_extension or level > 0):
         type_name = parent_table_name + "_" + type_name
+    if type_name.startswith("extension-definition"):
+        type_name = type_name[0:30]
+        type_name = type_name.replace("extension-definition-", "ext_def")
     table_name = canonicalize_table_name(type_name, schema_name)
     object_table = data_sink.tables_dictionary[table_name]
     sub_insert_statements = list()

From 5c38b11bb465c4948d340c3bb2c870212ce94aa5 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Thu, 4 Apr 2024 08:53:39 -0400
Subject: [PATCH 048/132] Adding enumproperty to simple type and edit
 openvocabproperty

---
 stix2/properties.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index a535cd02..014ad67b 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -406,7 +406,7 @@ def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
 
         simple_types = [
             BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty,
-            TimestampProperty, ReferenceProperty,
+            TimestampProperty, ReferenceProperty, EnumProperty,
         ]
         if not valid_types:
             valid_types = [Property]
@@ -722,7 +722,11 @@ def __init__(self, allowed, **kwargs):
         self.allowed = allowed
         super(EnumProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom, strict=False):
+        if not isinstance(value, str):
+            if strict is True:
+                raise ValueError("Must be a string.")
+
         cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom)
 
         if cleaned_value not in self.allowed:
@@ -743,7 +747,7 @@ def __init__(self, allowed, **kwargs):
             allowed = [allowed]
         self.allowed = allowed
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom, strict=False):
         cleaned_value, _ = super(OpenVocabProperty, self).clean(
             value, allow_custom,
         )

From 3042ddc6b2323c21457f40774e27b70b387cec42 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 4 Apr 2024 14:45:34 -0400
Subject: [PATCH 049/132] handle when the extension needs the id from the
 object

---
 stix2/datastore/relational_db/input_creation.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 8039fedb..1b720c8e 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -167,7 +167,7 @@ def generate_insert_information(self, name, stix_object, level=0, is_extension=F
         table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
         for idx, item in enumerate(stix_object[name]):
             bindings = {
-                "id": stix_object["id"],
+                "id": stix_object["id"] if id in stix_object else foreign_key_value,
                 "ref_id": item,
             }
             insert_statements.append(insert(table).values(bindings))

From 06b3d5af43531cdd67106c686b31214d720b4922 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 7 Apr 2024 14:30:38 -0400
Subject: [PATCH 050/132] fix fk on windows-pebinary

---
 stix2/datastore/relational_db/input_creation.py | 2 +-
 stix2/datastore/relational_db/table_creation.py | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 1b720c8e..7a24b55c 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -89,7 +89,7 @@ def generate_insert_information(self, name, stix_object, data_sink=None, table_n
     input_statements = list()
     for ex_name, ex in stix_object["extensions"].items():
         # ignore new extensions - they have no properties
-        if ex.extension_type and not ex.extension_type.startswith("new"):
+        if ex.extension_type is None or not ex.extension_type.startswith("new"):
             if ex_name.startswith("extension-definition"):
                 ex_name = ex_name[0:30]
                 ex_name = ex_name.replace("extension-definition-", "ext_def")
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 0b6e605d..0f3e9327 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -72,7 +72,7 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
     columns = list()
     # special case, perhaps because its a single embedded object with hashes, and not a list of embedded object
     # making the parent table's primary key does seem to worl
-    if table_name == "windows-pebinary-ext_WindowsPEOptionalHeaderType":
+    if False:  #  table_name == "windows-pebinary-ext_WindowsPEOptionalHeaderType":
         columns.append(
             Column(
                 "id",
@@ -737,7 +737,7 @@ def generate_object_table(
                         ondelete="CASCADE",
                     ),
                     # if it is a not list, then it is a single embedded object, and the primary key is unique
-                    # primary_key=not is_list
+                    primary_key=not is_list
                 )
         elif level > 0 and is_embedded_object:
             column = Column(

From f604a7637470c2ed0274c03fa6864745fdb66b8f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 7 Apr 2024 14:34:26 -0400
Subject: [PATCH 051/132] flaky

---
 .../datastore/relational_db/table_creation.py | 35 ++++++-------------
 1 file changed, 11 insertions(+), 24 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 0f3e9327..6711fda3 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -72,32 +72,19 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
     columns = list()
     # special case, perhaps because its a single embedded object with hashes, and not a list of embedded object
     # making the parent table's primary key does seem to worl
-    if False:  #  table_name == "windows-pebinary-ext_WindowsPEOptionalHeaderType":
-        columns.append(
-            Column(
-                "id",
-                key_type,
-                # ForeignKey(
-                #     canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
-                #     ondelete="CASCADE",
-                # ),
 
-                nullable=False,
+    columns.append(
+        Column(
+            "id",
+            key_type,
+            ForeignKey(
+                canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
+                ondelete="CASCADE",
             ),
-        )
-    else:
-        columns.append(
-            Column(
-                "id",
-                key_type,
-                ForeignKey(
-                    canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
-                    ondelete="CASCADE",
-                ),
 
-                nullable=False,
-            ),
-        )
+            nullable=False,
+        ),
+    )
     columns.append(
         Column(
             "hash_name",
@@ -737,7 +724,7 @@ def generate_object_table(
                         ondelete="CASCADE",
                     ),
                     # if it is a not list, then it is a single embedded object, and the primary key is unique
-                    primary_key=not is_list
+                    primary_key=not is_list,
                 )
         elif level > 0 and is_embedded_object:
             column = Column(

From cada2687e072323b4b62e8ef3b54c6f00c40ec14 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Sun, 7 Apr 2024 20:49:37 -0400
Subject: [PATCH 052/132] adding postgres authentication and fixing store.get
 bug

---
 stix2/test/v21/test_url.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stix2/test/v21/test_url.py b/stix2/test/v21/test_url.py
index 1832134a..ffa6ab15 100644
--- a/stix2/test/v21/test_url.py
+++ b/stix2/test/v21/test_url.py
@@ -8,12 +8,12 @@
 
 url_dict = {
     "type": "url",
-    "id": "url--c1477287-23ac-5971-a010-5c287877fa60",
+    "id": "url--a5477287-23ac-5971-a010-5c287877fa60",
     "value" : "https://example.com/research/index.html"
 }
 
 store = RelationalDBStore(
-        "postgresql://localhost/stix-data-sink",
+        "postgresql://postgres:admin@localhost/postgres",
         False,
         None,
         True
@@ -23,7 +23,7 @@ def test_url():
     store.sink.generate_stix_schema()
     url_stix_object = stix2.parse(url_dict)
     store.add(url_stix_object)
-    read_obj = json.loads(store.get(url_stix_object).serialize())
+    read_obj = json.loads(store.get(url_stix_object['id']).serialize())
 
     for attrib in url_dict.keys():
         assert url_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From b80b77f4da03457b636fed3be44950a89465442e Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 9 Apr 2024 12:26:16 -0400
Subject: [PATCH 053/132] adding test file for artifacts

---
 stix2/test/v21/test_artifact.py | 59 +++++++++++++++++++++++++++++++++
 1 file changed, 59 insertions(+)
 create mode 100644 stix2/test/v21/test_artifact.py

diff --git a/stix2/test/v21/test_artifact.py b/stix2/test/v21/test_artifact.py
new file mode 100644
index 00000000..ecb08cc0
--- /dev/null
+++ b/stix2/test/v21/test_artifact.py
@@ -0,0 +1,59 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+basic_artifact_dict = {
+    "type": "artifact",
+    "spec_version": "2.1",
+    "id": "artifact--cb17bcf8-9846-5ab4-8662-75c1bf6e63ee",
+    "mime_type": "image/jpeg",
+    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh"
+}
+
+encrypted_artifact_dict = {
+    "type": "artifact",
+    "spec_version": "2.1",
+    "id": "artifact--3157f78d-7d16-5092-99fe-ecff58408b02",
+    "mime_type": "application/zip",
+    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhbiB1bnNhZmUgbWFsd2FyZSBiaW5hcnkh",
+    "hashes": {
+        "MD5": "6b885a1e1d42c0ca66e5f8a17e5a5d29",
+        "SHA-256": "3eea3c4819e9d387ff6809f13dde5426b9466285b7d923016b2842a13eb2888b"
+    },
+    "encryption_algorithm": "mime-type-indicated",
+    "decryption_key": "My voice is my passport"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_basic_artifact():
+    store.sink.generate_stix_schema()
+    artifact_stix_object = stix2.parse(basic_artifact_dict)
+    store.add(artifact_stix_object)
+    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
+
+    for attrib in basic_artifact_dict.keys():
+        assert basic_artifact_dict[attrib] == read_obj[attrib]
+
+def test_encrypted_artifact():
+    store.sink.generate_stix_schema()
+    artifact_stix_object = stix2.parse(encrypted_artifact_dict)
+    store.add(artifact_stix_object)
+    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
+
+    for attrib in encrypted_artifact_dict.keys():
+        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
+
+
+def main():
+    test_encrypted_artifact()
+
+main()
\ No newline at end of file

From dce689a6a05e163cf9908caf4311db2d99286fbe Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 9 Apr 2024 13:47:22 -0400
Subject: [PATCH 054/132] fix multiple schema names

---
 .../datastore/relational_db/input_creation.py | 25 +++++++++----------
 1 file changed, 12 insertions(+), 13 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 7a24b55c..4e45e405 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -97,7 +97,7 @@ def generate_insert_information(self, name, stix_object, data_sink=None, table_n
                 "id": stix_object["id"],
                 "ext_table_name": canonicalize_table_name(ex_name, schema_name),
             }
-            ex_table = data_sink.tables_dictionary[table_name + "_" + "extensions"]
+            ex_table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + "extensions", schema_name)]
             input_statements.append(insert(ex_table).values(bindings))
             input_statements.extend(
                 generate_insert_for_sub_object(
@@ -158,13 +158,14 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(ListProperty)
-def generate_insert_information(self, name, stix_object, level=0, is_extension=False, foreign_key_value=None, **kwargs):  # noqa: F811
+def generate_insert_information(self, name, stix_object, level=0, is_extension=False,
+                                foreign_key_value=None, schema_name=None, **kwargs):  # noqa: F811
     data_sink = kwargs.get("data_sink")
     table_name = kwargs.get("table_name")
     if isinstance(self.contained, ReferenceProperty):
         insert_statements = list()
 
-        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
+        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
         for idx, item in enumerate(stix_object[name]):
             bindings = {
                 "id": stix_object["id"] if id in stix_object else foreign_key_value,
@@ -177,7 +178,7 @@ def generate_insert_information(self, name, stix_object, level=0, is_extension=F
         for value in stix_object[name]:
             with data_sink.database_connection.begin() as trans:
                 next_id = trans.execute(data_sink.sequence)
-            table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name)]
+            table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
             bindings = {
                 "id": foreign_key_value,
                 "ref_id": next_id,
@@ -188,7 +189,7 @@ def generate_insert_information(self, name, stix_object, level=0, is_extension=F
                     data_sink,
                     value,
                     table_name + "_" + name + "_" + self.contained.type.__name__,
-                    None,
+                    schema_name,
                     next_id,
                     level,
                     True,
@@ -355,8 +356,6 @@ def generate_insert_for_sub_object(
     if type_name.startswith("extension-definition"):
         type_name = type_name[0:30]
         type_name = type_name.replace("extension-definition-", "ext_def")
-    table_name = canonicalize_table_name(type_name, schema_name)
-    object_table = data_sink.tables_dictionary[table_name]
     sub_insert_statements = list()
     for name, prop in stix_object._properties.items():
         if name in stix_object:
@@ -364,14 +363,14 @@ def generate_insert_for_sub_object(
                 name,
                 stix_object,
                 data_sink=data_sink,
-                table_name=table_name if isinstance(prop, (DictionaryProperty, ListProperty)) else parent_table_name,
-                schema_name=None,
+                table_name=type_name if isinstance(prop, (DictionaryProperty, ListProperty)) else parent_table_name,
+                schema_name=schema_name,
                 foreign_key_value=foreign_key_value,
                 is_embedded_object=is_embedded_object,
                 is_list=is_list,
                 level=level+1,
                 is_extension=is_extension,
-                parent_table_name=table_name,
+                parent_table_name=type_name,
             )
             if isinstance(result, dict):
                 bindings.update(result)
@@ -381,6 +380,7 @@ def generate_insert_for_sub_object(
                 raise ValueError("wrong type" + result)
     if foreign_key_value:
         bindings["id"] = foreign_key_value
+    object_table = data_sink.tables_dictionary[canonicalize_table_name(type_name, schema_name)]
     insert_statements.append(insert(object_table).values(bindings))
     insert_statements.extend(sub_insert_statements)
     return insert_statements
@@ -394,8 +394,6 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
     else:
         core_properties = SDO_COMMON_PROPERTIES
     type_name = stix_object["type"]
-    table_name = canonicalize_table_name(type_name, schema_name)
-    object_table = data_sink.tables_dictionary[table_name]
     insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
     if "id" in stix_object:
         foreign_key_value = stix_object["id"]
@@ -407,7 +405,7 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
             result = prop.generate_insert_information(
                 name, stix_object,
                 data_sink=data_sink,
-                table_name=table_name,
+                table_name=type_name,
                 schema_name=schema_name,
                 parent_table_name=type_name,
                 level=level,
@@ -420,6 +418,7 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
             else:
                 raise ValueError("wrong type" + result)
 
+    object_table = data_sink.tables_dictionary[canonicalize_table_name(type_name, schema_name)]
     insert_statements.append(insert(object_table).values(bindings))
     insert_statements.extend(sub_insert_statements)
 

From 8529c50add609ae41df537fcb983d39ca342f9ad Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 9 Apr 2024 15:14:45 -0400
Subject: [PATCH 055/132] skipping over hashes check, removing main function

---
 stix2/test/v21/test_artifact.py | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/stix2/test/v21/test_artifact.py b/stix2/test/v21/test_artifact.py
index ecb08cc0..9c231a29 100644
--- a/stix2/test/v21/test_artifact.py
+++ b/stix2/test/v21/test_artifact.py
@@ -8,7 +8,7 @@
 basic_artifact_dict = {
     "type": "artifact",
     "spec_version": "2.1",
-    "id": "artifact--cb17bcf8-9846-5ab4-8662-75c1bf6e63ee",
+    "id": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
     "mime_type": "image/jpeg",
     "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh"
 }
@@ -16,7 +16,7 @@
 encrypted_artifact_dict = {
     "type": "artifact",
     "spec_version": "2.1",
-    "id": "artifact--3157f78d-7d16-5092-99fe-ecff58408b02",
+    "id": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
     "mime_type": "application/zip",
     "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhbiB1bnNhZmUgbWFsd2FyZSBiaW5hcnkh",
     "hashes": {
@@ -50,10 +50,6 @@ def test_encrypted_artifact():
     read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
 
     for attrib in encrypted_artifact_dict.keys():
-        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
-
-
-def main():
-    test_encrypted_artifact()
-
-main()
\ No newline at end of file
+        if attrib == 'hashes': # TODO hashes are saved to separate table, functionality to retrieve is WIP
+            continue
+        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From eb989cd98dab0cf8e0ca136d3c4d52743ae5d64a Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 9 Apr 2024 15:25:21 -0400
Subject: [PATCH 056/132] adding autonomous system test

---
 stix2/test/v21/test_autonomous_system.py | 30 ++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 stix2/test/v21/test_autonomous_system.py

diff --git a/stix2/test/v21/test_autonomous_system.py b/stix2/test/v21/test_autonomous_system.py
new file mode 100644
index 00000000..aa4b5465
--- /dev/null
+++ b/stix2/test/v21/test_autonomous_system.py
@@ -0,0 +1,30 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+
+as_dict = {
+    "type": "autonomous-system",
+    "spec_version": "2.1",
+    "id": "autonomous-system--f822c34b-98ae-597f-ade5-27dc241e8c74",
+    "number": 15139,
+    "name": "Slime Industries",
+    "rir": "ARIN"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_autonomous_system():
+    store.sink.generate_stix_schema()
+    as_obj = stix2.parse(as_dict)
+    store.add(as_obj)
+    read_obj = json.loads(store.get(as_obj['id']).serialize())
+
+    for attrib in as_dict.keys():
+        assert as_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From 2ab4699c6e573780dcfd1ae45bc675bec5e9f2bf Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 9 Apr 2024 15:29:17 -0400
Subject: [PATCH 057/132] adding domain name test file

---
 stix2/test/v21/test_domain_name.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)
 create mode 100644 stix2/test/v21/test_domain_name.py

diff --git a/stix2/test/v21/test_domain_name.py b/stix2/test/v21/test_domain_name.py
new file mode 100644
index 00000000..e57a0625
--- /dev/null
+++ b/stix2/test/v21/test_domain_name.py
@@ -0,0 +1,28 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+
+domain_name_dict = {    
+    "type": "domain-name",
+    "spec_version": "2.1",
+    "id": "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5",
+    "value": "example.com",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_autonomous_system():
+    store.sink.generate_stix_schema()
+    domain_name_obj = stix2.parse(domain_name_dict)
+    store.add(domain_name_obj)
+    read_obj = json.loads(store.get(domain_name_obj['id']).serialize())
+
+    for attrib in domain_name_dict.keys():
+        assert domain_name_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From 3e91526602b6821c8f13ca7761c2e269f706f24f Mon Sep 17 00:00:00 2001
From: Robert Thew <rthew@mitre.org>
Date: Tue, 9 Apr 2024 15:47:17 -0400
Subject: [PATCH 058/132] Added database existence checks and clear_tables
 function

---
 .../datastore/relational_db/relational_db.py  | 50 ++++++++++++-------
 .../relational_db/relational_db_testing.py    | 26 ++++++----
 2 files changed, 47 insertions(+), 29 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index db217eb4..48ff97f6 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,5 +1,6 @@
-from sqlalchemy import MetaData, create_engine, select
+from sqlalchemy import MetaData, create_engine, select, delete
 from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
+from sqlalchemy_utils import database_exists, create_database
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
@@ -75,25 +76,30 @@ def __init__(
                 them.
         """
         database_connection = create_engine(database_connection_url)
+        self.database_exists = database_exists(database_connection.url)
+        if not self.database_exists and instantiate_database:
+            create_database(database_connection_url)
+            self.database_exists = database_exists(database_connection.url)
 
-        self.metadata = MetaData()
-        create_table_objects(
-            self.metadata, stix_object_classes,
-        )
+        if self.database_exists:
+            self.metadata = MetaData()
+            create_table_objects(
+                self.metadata, stix_object_classes,
+            )
 
-        super().__init__(
-            source=RelationalDBSource(
-                database_connection,
-                metadata=self.metadata,
-            ),
-            sink=RelationalDBSink(
-                database_connection,
-                allow_custom=allow_custom,
-                version=version,
-                instantiate_database=instantiate_database,
-                metadata=self.metadata,
-            ),
-        )
+            super().__init__(
+                source=RelationalDBSource(
+                    database_connection,
+                    metadata=self.metadata,
+                ),
+                sink=RelationalDBSink(
+                    database_connection,
+                    allow_custom=allow_custom,
+                    version=version,
+                    instantiate_database=instantiate_database,
+                    metadata=self.metadata,
+                ),
+            )
 
 
 class RelationalDBSink(DataSink):
@@ -178,6 +184,14 @@ def insert_object(self, stix_object):
                 trans.execute(stmt)
             trans.commit()
 
+    def clear_tables(self):
+        tables = list(reversed(self.metadata.sorted_tables))
+        with self.database_connection.begin() as trans:
+            for table in tables:
+                delete_stmt = delete(table)
+                print(f'delete_stmt: {delete_stmt}')
+                trans.execute(delete_stmt)
+
 
 class RelationalDBSource(DataSource):
     def __init__(
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 3e3cccce..7b21cdc1 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -22,13 +22,13 @@
 )
 
 s = stix2.v21.Software(
-        id="software--28897173-7314-4eec-b1cf-2c625b635bf6",
-        name="Word",
-        cpe="cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
-        swid="com.acme.rms-ce-v4-1-5-0",
-        version="2002",
-        languages=["c", "lisp"],
-        vendor="Microsoft",
+    id="software--28897173-7314-4eec-b1cf-2c625b635bf6",
+    name="Word",
+    cpe="cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
+    swid="com.acme.rms-ce-v4-1-5-0",
+    version="2002",
+    languages=["c", "lisp"],
+    vendor="Microsoft",
 )
 
 
@@ -103,12 +103,16 @@ def main():
         True,
         stix2.Directory,
     )
-    store.sink.generate_stix_schema()
+    if store.database_exists:
+        store.sink.generate_stix_schema()
+        store.sink.clear_tables()
 
-    store.add(directory_stix_object)
+        store.add(directory_stix_object)
 
-    read_obj = store.get(directory_stix_object.id)
-    print(read_obj)
+        read_obj = store.get(directory_stix_object.id)
+        print(read_obj)
+    else:
+        print("database does not exist")
 
 
 if __name__ == '__main__':

From 8e83c31fb3854e20106c43914c4c4d04a416f8db Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Tue, 9 Apr 2024 17:15:17 -0400
Subject: [PATCH 059/132] adding directory test file

---
 stix2/test/v21/test_directory.py | 36 ++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 stix2/test/v21/test_directory.py

diff --git a/stix2/test/v21/test_directory.py b/stix2/test/v21/test_directory.py
new file mode 100644
index 00000000..98c3122c
--- /dev/null
+++ b/stix2/test/v21/test_directory.py
@@ -0,0 +1,36 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+
+# Ctime and mtime will return as stix2 objects from the DB get function so comparison doesn't work for those
+directory_dict = {
+    "type": "directory",
+    "spec_version": "2.1",
+    "id": "directory--67c0a9b0-520d-545d-9094-1a08ddf46b05",
+    "path": "C:\\Windows\\System32",
+    "path_enc": "cGF0aF9lbmM",
+    "contains_refs": [
+        "directory--94c0a9b0-520d-545d-9094-1a08ddf46b05",
+        "file--95c0a9b0-520d-545d-9094-1a08ddf46b05"
+    ]
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_directory():
+    store.sink.generate_stix_schema()
+    directory_obj = stix2.parse(directory_dict)
+    store.add(directory_obj)
+    read_obj = json.loads(store.get(directory_obj['id']).serialize())
+
+    for attrib in directory_dict.keys():
+        if attrib == "contains_refs": # TODO remove skip once we can pull from table join
+            continue
+        assert directory_dict[attrib] == read_obj[attrib]
\ No newline at end of file

From 917a4cc449bf0aa1589363ae4d8425ed9291bcda Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Wed, 10 Apr 2024 12:59:17 -0400
Subject: [PATCH 060/132] passing strict to superclass of enum and openvocab

---
 stix2/properties.py | 20 +++++++++-----------
 1 file changed, 9 insertions(+), 11 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 014ad67b..55358d41 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -723,11 +723,8 @@ def __init__(self, allowed, **kwargs):
         super(EnumProperty, self).__init__(**kwargs)
 
     def clean(self, value, allow_custom, strict=False):
-        if not isinstance(value, str):
-            if strict is True:
-                raise ValueError("Must be a string.")
 
-        cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom)
+        cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom, strict)
 
         if cleaned_value not in self.allowed:
             raise ValueError("value '{}' is not valid for this enumeration.".format(cleaned_value))
@@ -749,19 +746,20 @@ def __init__(self, allowed, **kwargs):
 
     def clean(self, value, allow_custom, strict=False):
         cleaned_value, _ = super(OpenVocabProperty, self).clean(
-            value, allow_custom,
+            value, allow_custom, strict
         )
 
         # Disabled: it was decided that enforcing this is too strict (might
         # break too much user code).  Revisit when we have the capability for
         # more granular config settings when creating objects.
         #
-        # has_custom = cleaned_value not in self.allowed
-        #
-        # if not allow_custom and has_custom:
-        #     raise CustomContentError(
-        #         "custom value in open vocab: '{}'".format(cleaned_value),
-        #     )
+        if strict is True:
+            has_custom = cleaned_value not in self.allowed
+            
+            if not allow_custom and has_custom:
+                raise CustomContentError(
+                    "custom value in open vocab: '{}'".format(cleaned_value),
+                )
 
         has_custom = False
 

From f402280f3feef869ab4f27ff7d04c57d3da0be4b Mon Sep 17 00:00:00 2001
From: Robert Thew <rthew@mitre.org>
Date: Wed, 10 Apr 2024 13:58:45 -0400
Subject: [PATCH 061/132] added parameters for database creation

---
 stix2/datastore/relational_db/relational_db.py         | 8 +++++---
 stix2/datastore/relational_db/relational_db_testing.py | 3 ++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 48ff97f6..aa1a391c 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,6 +1,6 @@
 from sqlalchemy import MetaData, create_engine, select, delete
 from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
-from sqlalchemy_utils import database_exists, create_database
+from sqlalchemy_utils import database_exists, create_database, drop_database
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
@@ -56,7 +56,7 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 class RelationalDBStore(DataStoreMixin):
     def __init__(
         self, database_connection_url, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes,
+        create_db=True, instantiate_database=True, *stix_object_classes,
     ):
         """
         Initialize this store.
@@ -77,7 +77,9 @@ def __init__(
         """
         database_connection = create_engine(database_connection_url)
         self.database_exists = database_exists(database_connection.url)
-        if not self.database_exists and instantiate_database:
+        if create_db:
+            if self.database_exists:
+                drop_database(database_connection_url)
             create_database(database_connection_url)
             self.database_exists = database_exists(database_connection.url)
 
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 7b21cdc1..0f326135 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -97,10 +97,11 @@ def file_example_with_PDFExt_Object():
 
 def main():
     store = RelationalDBStore(
-        "postgresql://localhost/stix-data-sink",
+        "postgresql://localhost/stix-data-sink4",
         False,
         None,
         True,
+        True,
         stix2.Directory,
     )
     if store.database_exists:

From 9ef379c159d38b162ca5902cae63159ba5ed1c5e Mon Sep 17 00:00:00 2001
From: Robert Thew <rthew@mitre.org>
Date: Wed, 10 Apr 2024 14:14:19 -0400
Subject: [PATCH 062/132] Fixed db URL

---
 stix2/datastore/relational_db/relational_db_testing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 0f326135..e948b929 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -97,7 +97,7 @@ def file_example_with_PDFExt_Object():
 
 def main():
     store = RelationalDBStore(
-        "postgresql://localhost/stix-data-sink4",
+        "postgresql://localhost/stix-data-sink",
         False,
         None,
         True,

From 4ef83608248749762d5d039895720ff47fae4ea3 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 13:25:08 -0400
Subject: [PATCH 063/132] updating directory test with ctime and mtime check

---
 stix2/test/v21/test_directory.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/stix2/test/v21/test_directory.py b/stix2/test/v21/test_directory.py
index 98c3122c..94ca93f9 100644
--- a/stix2/test/v21/test_directory.py
+++ b/stix2/test/v21/test_directory.py
@@ -4,11 +4,12 @@
 
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
-# Ctime and mtime will return as stix2 objects from the DB get function so comparison doesn't work for those
 directory_dict = {
     "type": "directory",
     "spec_version": "2.1",
-    "id": "directory--67c0a9b0-520d-545d-9094-1a08ddf46b05",
+    "id": "directory--17c909b1-521d-545d-9094-1a08ddf46b05",
+    "ctime": "2018-11-23T08:17:27.000Z",
+    "mtime": "2018-11-23T08:17:27.000Z",
     "path": "C:\\Windows\\System32",
     "path_enc": "cGF0aF9lbmM",
     "contains_refs": [
@@ -33,4 +34,8 @@ def test_directory():
     for attrib in directory_dict.keys():
         if attrib == "contains_refs": # TODO remove skip once we can pull from table join
             continue
-        assert directory_dict[attrib] == read_obj[attrib]
\ No newline at end of file
+        if attrib == "ctime" or attrib == "mtime": # convert both into stix2 date format for consistency
+            assert stix2.utils.parse_into_datetime(directory_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert directory_dict[attrib] == read_obj[attrib]
+

From baa778191baad60c471482fa02432d9e0109c2e1 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 13:39:13 -0400
Subject: [PATCH 064/132] adding email address test file

---
 stix2/test/v21/test_email_address.py | 31 ++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)
 create mode 100644 stix2/test/v21/test_email_address.py

diff --git a/stix2/test/v21/test_email_address.py b/stix2/test/v21/test_email_address.py
new file mode 100644
index 00000000..8ec4782c
--- /dev/null
+++ b/stix2/test/v21/test_email_address.py
@@ -0,0 +1,31 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+email_addr_dict = {
+    "type": "email-addr",
+    "spec_version": "2.1",
+    "id": "email-addr--2d77a846-6264-5d51-b586-e43822ea1ea3",
+    "value": "john@example.com",
+    "display_name": "John Doe",
+    "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_email_addr():
+    store.sink.generate_stix_schema()
+    email_addr_stix_object = stix2.parse(email_addr_dict)
+    store.add(email_addr_stix_object)
+    read_obj = json.loads(store.get(email_addr_stix_object['id']).serialize())
+
+    for attrib in email_addr_dict.keys():
+        assert email_addr_dict[attrib] == read_obj[attrib]

From 605c1814a9cfefd7fb655525f8981868703d1bac Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 14:25:10 -0400
Subject: [PATCH 065/132] adding test file for email messages

---
 stix2/test/v21/test_email_message.py | 122 +++++++++++++++++++++++++++
 1 file changed, 122 insertions(+)
 create mode 100644 stix2/test/v21/test_email_message.py

diff --git a/stix2/test/v21/test_email_message.py b/stix2/test/v21/test_email_message.py
new file mode 100644
index 00000000..bb17b30b
--- /dev/null
+++ b/stix2/test/v21/test_email_message.py
@@ -0,0 +1,122 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+email_msg_dict = {
+    "type": "email-message",
+    "spec_version": "2.1",
+    "id": "email-message--8c57a381-2a17-5e61-8754-5ef96efb286c",
+    "from_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5dda",
+    "sender_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5eeb",
+    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
+    "cc_refs": [
+        "email-addr--d2b3bf0c-f02a-51a1-8102-11aba7959868",
+        "email-addr--d3b3bf0c-f02a-51a1-8102-11aba7959868"
+    ],
+    "bcc_refs": [
+        "email-addr--d4b3bf0c-f02a-51a1-8102-11aba7959868",
+        "email-addr--d5b3bf0c-f02a-51a1-8102-11aba7959868"
+    ],
+    "message_id": "message01",
+    "is_multipart": False,
+    "date": "2004-04-19T12:22:23.000Z",
+    "subject": "Did you see this?",
+    "received_lines": [
+        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
+        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)"
+    ],
+    "additional_header_fields": {
+        "Reply-To": [
+            "steve@example.com",
+            "jane@example.com"
+        ]
+    },
+    "body": "message body",
+    "raw_email_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee"
+}
+
+multipart_email_msg_dict = {
+    "type": "email-message",
+    "spec_version": "2.1",
+    "id": "email-message--ef9b4b7f-14c8-5955-8065-020e0316b559",
+    "is_multipart": True,
+    "received_lines": [
+        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
+        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)"
+    ],
+    "content_type": "multipart/mixed",
+    "date": "2016-06-19T14:20:40.000Z",
+    "from_ref": "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed",
+    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
+    "cc_refs": ["email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194"],
+    "subject": "Check out this picture of a cat!",
+    "additional_header_fields": {
+        "Content-Disposition": "inline",
+        "X-Mailer": "Mutt/1.5.23",
+        "X-Originating-IP": "198.51.100.3"
+    },
+    "body_multipart": [
+        {
+        "content_type": "text/plain; charset=utf-8",
+        "content_disposition": "inline",
+        "body": "Cats are funny!"
+        },
+        {
+        "content_type": "image/png",
+        "content_disposition": "attachment; filename=\"tabby.png\"",
+        "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5"
+        },
+        {
+        "content_type": "application/zip",
+        "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
+        "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5"
+        }
+    ]
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_email_msg():
+    store.sink.generate_stix_schema()
+    email_msg_stix_object = stix2.parse(email_msg_dict)
+    store.add(email_msg_stix_object)
+    read_obj = json.loads(store.get(email_msg_stix_object['id']).serialize())
+
+    for attrib in email_msg_dict.keys():
+        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
+            or attrib == "additional_header_fields": # join multiple tables not implemented yet
+            continue
+        if attrib == "date":
+            assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        print(email_msg_dict[attrib], type(email_msg_dict[attrib]))
+        print(read_obj[attrib], type(read_obj[attrib]))
+        assert email_msg_dict[attrib] == read_obj[attrib]
+
+def test_multipart_email_msg():
+    store.sink.generate_stix_schema()
+    multipart_email_msg_stix_object = stix2.parse(multipart_email_msg_dict)
+    store.add(multipart_email_msg_stix_object)
+    read_obj = json.loads(store.get(multipart_email_msg_stix_object['id']).serialize())
+
+    for attrib in multipart_email_msg_dict.keys():
+        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
+            or attrib == "additional_header_fields" or attrib == "body_multipart": # join multiple tables not implemented yet
+            continue
+        if attrib == "date":
+            assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        print(multipart_email_msg_dict[attrib], type(multipart_email_msg_dict[attrib]))
+        print(read_obj[attrib], type(read_obj[attrib]))
+        assert multipart_email_msg_dict[attrib] == read_obj[attrib]
+

From 9646a59033843d3a578a9221b87dcc47ed61087a Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 14:27:51 -0400
Subject: [PATCH 066/132] removing print statements in email message test

---
 stix2/test/v21/test_email_message.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/stix2/test/v21/test_email_message.py b/stix2/test/v21/test_email_message.py
index bb17b30b..aef25458 100644
--- a/stix2/test/v21/test_email_message.py
+++ b/stix2/test/v21/test_email_message.py
@@ -99,8 +99,6 @@ def test_email_msg():
         if attrib == "date":
             assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
-        print(email_msg_dict[attrib], type(email_msg_dict[attrib]))
-        print(read_obj[attrib], type(read_obj[attrib]))
         assert email_msg_dict[attrib] == read_obj[attrib]
 
 def test_multipart_email_msg():
@@ -116,7 +114,5 @@ def test_multipart_email_msg():
         if attrib == "date":
             assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
-        print(multipart_email_msg_dict[attrib], type(multipart_email_msg_dict[attrib]))
-        print(read_obj[attrib], type(read_obj[attrib]))
         assert multipart_email_msg_dict[attrib] == read_obj[attrib]
 

From 8015c2c028aff47440c1a1bbe4d59e6a88e53a81 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 17:35:16 -0400
Subject: [PATCH 067/132] adding tests for ipv4 and ipv6

---
 stix2/test/v21/test_ipv4_ipv6.py | 47 ++++++++++++++++++++++++++++++++
 1 file changed, 47 insertions(+)
 create mode 100644 stix2/test/v21/test_ipv4_ipv6.py

diff --git a/stix2/test/v21/test_ipv4_ipv6.py b/stix2/test/v21/test_ipv4_ipv6.py
new file mode 100644
index 00000000..73f45541
--- /dev/null
+++ b/stix2/test/v21/test_ipv4_ipv6.py
@@ -0,0 +1,47 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+ipv4_dict = {
+    "type": "ipv4-addr",
+    "spec_version": "2.1",
+    "id": "ipv4-addr--ff26c255-6336-5bc5-b98d-13d6226742dd",
+    "value": "198.51.100.3"
+}
+
+ipv6_dict = { 
+    "type": "ipv6-addr",
+    "spec_version": "2.1",
+    "id": "ipv6-addr--1e61d36c-a26c-53b7-a80f-2a00161c96b1",
+    "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_ipv4():
+    store.sink.generate_stix_schema()
+    ipv4_stix_object = stix2.parse(ipv4_dict)
+    store.add(ipv4_stix_object)
+    read_obj = store.get(ipv4_stix_object['id'])
+
+    for attrib in ipv4_dict.keys():
+        assert ipv4_dict[attrib] == read_obj[attrib]
+
+
+def test_ipv6():
+    store.sink.generate_stix_schema()
+    ipv6_stix_object = stix2.parse(ipv6_dict)
+    store.add(ipv6_stix_object)
+    read_obj = store.get(ipv6_stix_object['id'])
+
+    for attrib in ipv6_dict.keys():
+        assert ipv6_dict[attrib] == read_obj[attrib]
+

From 50412eece010daed4669d47c1896d1f64d896345 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Fri, 12 Apr 2024 17:40:48 -0400
Subject: [PATCH 068/132] adding mutext test file

---
 stix2/test/v21/test_mutex.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 stix2/test/v21/test_mutex.py

diff --git a/stix2/test/v21/test_mutex.py b/stix2/test/v21/test_mutex.py
new file mode 100644
index 00000000..0ef4a082
--- /dev/null
+++ b/stix2/test/v21/test_mutex.py
@@ -0,0 +1,30 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+mutex_dict = {
+    "type": "mutex",
+    "spec_version": "2.1",
+    "id": "mutex--fba44954-d4e4-5d3b-814c-2b17dd8de300",
+    "name": "__CLEANSWEEP__"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_mutex():
+    store.sink.generate_stix_schema()
+    mutex_stix_object = stix2.parse(mutex_dict)
+    store.add(mutex_stix_object)
+    read_obj = store.get(mutex_stix_object['id'])
+
+    for attrib in mutex_dict.keys():
+        assert mutex_dict[attrib] == read_obj[attrib]
+

From a2646ff426f8d61f388ffc52cca187b2143e8cdf Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:12:33 -0400
Subject: [PATCH 069/132] adding tests for file and network_traffic objects

---
 stix2/test/v21/test_file.py            | 52 ++++++++++++++++++++++
 stix2/test/v21/test_network_traffic.py | 60 ++++++++++++++++++++++++++
 2 files changed, 112 insertions(+)
 create mode 100644 stix2/test/v21/test_file.py
 create mode 100644 stix2/test/v21/test_network_traffic.py

diff --git a/stix2/test/v21/test_file.py b/stix2/test/v21/test_file.py
new file mode 100644
index 00000000..32430858
--- /dev/null
+++ b/stix2/test/v21/test_file.py
@@ -0,0 +1,52 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+# errors when adding magic_number_hex to store, so ignoring for now
+
+file_dict = {
+    "type": "file",
+    "spec_version": "2.1",
+    "id": "file--66156fad-2a7d-5237-bbb4-ba1912887cfe",
+    "hashes": {
+        "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a"
+    },
+    "parent_directory_ref": "directory--93c0a9b0-520d-545d-9094-1a08ddf46b05",
+    "name": "qwerty.dll",
+    "size": 25536,
+    "name_enc": "windows-1252",
+    "mime_type": "application/msword",
+    "ctime": "2018-11-23T08:17:27.000Z",
+    "mtime": "2018-11-23T08:17:27.000Z",
+    "atime": "2018-11-23T08:17:27.000Z",
+    "contains_refs": [
+        "file--77156fad-2a0d-5237-bba4-ba1912887cfe"
+    ],
+    "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_file():
+    store.sink.generate_stix_schema()
+    file_stix_object = stix2.parse(file_dict)
+    store.add(file_stix_object)
+    read_obj = store.get(file_stix_object['id'])
+    read_obj = json.loads(store.get(file_stix_object['id']).serialize())
+
+    for attrib in file_dict.keys():
+        if attrib == "contains_refs" or attrib == "hashes": # join multiple tables not implemented yet
+            continue
+        if attrib == "ctime" or attrib == "mtime" or attrib == "atime":
+            assert stix2.utils.parse_into_datetime(file_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert file_dict[attrib] == read_obj[attrib]
+
diff --git a/stix2/test/v21/test_network_traffic.py b/stix2/test/v21/test_network_traffic.py
new file mode 100644
index 00000000..3ca1c32c
--- /dev/null
+++ b/stix2/test/v21/test_network_traffic.py
@@ -0,0 +1,60 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+# ipfix property results in a unconsumed value error with the store add
+
+network_traffic_dict = {
+    "type": "network-traffic",
+    "spec_version": "2.1",
+    "id": "network-traffic--631d7bb1-6bbc-53a6-a6d4-f3c2d35c2734",
+    "src_ref": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53",
+    "dst_ref": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
+    "start": "2018-11-23T08:17:27.000Z",
+    "end": "2018-11-23T08:18:27.000Z",
+    "is_active": False,
+    "src_port": 1000,
+    "dst_port": 1000,
+    "protocols": [
+        "ipv4",
+        "tcp"
+    ],
+    "src_byte_count": 147600,
+    "dst_byte_count": 147600,
+    "src_packets": 100,
+    "dst_packets": 100,
+    "src_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
+    "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03", 
+    "encapsulates_refs" : [
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4"
+    ],
+    "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_network_traffic():
+    store.sink.generate_stix_schema()
+    network_traffic_stix_object = stix2.parse(network_traffic_dict)
+    store.add(network_traffic_stix_object)
+    read_obj = store.get(network_traffic_stix_object['id'])
+
+    for attrib in network_traffic_dict.keys():
+        if attrib == "encapsulates_refs": # multiple table join not implemented
+            continue 
+        if attrib == "start" or attrib == "end":
+            assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert network_traffic_dict[attrib] == read_obj[attrib]
+
+
+

From d74253e88638aa2e70f01be275f4fc841d21c13e Mon Sep 17 00:00:00 2001
From: Robert Thew <rthew@mitre.org>
Date: Mon, 15 Apr 2024 14:19:23 -0400
Subject: [PATCH 070/132] Adjusted database creation options

---
 .../datastore/relational_db/relational_db.py  | 66 +++++++++++--------
 .../relational_db/relational_db_testing.py    |  5 +-
 2 files changed, 41 insertions(+), 30 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index aa1a391c..0fa674f5 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -56,7 +56,7 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 class RelationalDBStore(DataStoreMixin):
     def __init__(
         self, database_connection_url, allow_custom=True, version=None,
-        create_db=True, instantiate_database=True, *stix_object_classes,
+        instantiate_database=True, force_recreate=False, *stix_object_classes,
     ):
         """
         Initialize this store.
@@ -68,6 +68,8 @@ def __init__(
             version: TODO: unused so far
             instantiate_database: Whether tables, etc should be created in the
                 database (only necessary the first time)
+            force_recreate: Drops old database and creates new one (useful if
+                the schema has changed and the tables need to be updated)
             *stix_object_classes: STIX object classes to map into table schemas
                 (and ultimately database tables, if instantiation is desired).
                 This can be used to limit which table schemas are created, if
@@ -76,38 +78,32 @@ def __init__(
                 them.
         """
         database_connection = create_engine(database_connection_url)
-        self.database_exists = database_exists(database_connection.url)
-        if create_db:
-            if self.database_exists:
-                drop_database(database_connection_url)
-            create_database(database_connection_url)
-            self.database_exists = database_exists(database_connection.url)
 
-        if self.database_exists:
-            self.metadata = MetaData()
-            create_table_objects(
-                self.metadata, stix_object_classes,
-            )
+        self.metadata = MetaData()
+        create_table_objects(
+            self.metadata, stix_object_classes,
+        )
 
-            super().__init__(
-                source=RelationalDBSource(
-                    database_connection,
-                    metadata=self.metadata,
-                ),
-                sink=RelationalDBSink(
-                    database_connection,
-                    allow_custom=allow_custom,
-                    version=version,
-                    instantiate_database=instantiate_database,
-                    metadata=self.metadata,
-                ),
-            )
+        super().__init__(
+            source=RelationalDBSource(
+                database_connection,
+                metadata=self.metadata,
+            ),
+            sink=RelationalDBSink(
+                database_connection,
+                allow_custom=allow_custom,
+                version=version,
+                instantiate_database=instantiate_database,
+                force_recreate=force_recreate,
+                metadata=self.metadata,
+            ),
+        )
 
 
 class RelationalDBSink(DataSink):
     def __init__(
         self, database_connection_or_url, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes, metadata=None,
+        instantiate_database=True, force_recreate=False, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this sink.  Only one of stix_object_classes and metadata
@@ -119,8 +115,10 @@ def __init__(
             allow_custom: Whether custom content is allowed when processing
                 dict content to be added to the sink
             version: TODO: unused so far
-            instantiate_database: Whether tables, etc should be created in the
-                database (only necessary the first time)
+            instantiate_database: Whether the database, tables, etc should be
+                created (only necessary the first time)
+            force_recreate: Drops old database and creates new one (useful if
+                the schema has changed and the tables need to be updated)
             *stix_object_classes: STIX object classes to map into table schemas
                 (and ultimately database tables, if instantiation is desired).
                 This can be used to limit which table schemas are created, if
@@ -140,6 +138,10 @@ def __init__(
         else:
             self.database_connection = database_connection_or_url
 
+        self.database_exists = database_exists(self.database_connection.url)
+        if force_recreate:
+            self._create_database()
+
         if metadata:
             self.metadata = metadata
         else:
@@ -156,6 +158,8 @@ def __init__(
             self.tables_dictionary[canonicalize_table_name(t.name, t.schema)] = t
 
         if instantiate_database:
+            if not self.database_exists:
+                self._create_database()
             self._create_schemas()
             self._instantiate_database()
 
@@ -169,6 +173,12 @@ def _create_schemas(self):
     def _instantiate_database(self):
         self.metadata.create_all(self.database_connection)
 
+    def _create_database(self):
+        if self.database_exists:
+            drop_database(self.database_connection.url)
+        create_database(self.database_connection.url)
+        self.database_exists = database_exists(self.database_connection.url)
+
     def generate_stix_schema(self):
         for t in self.metadata.tables.values():
             print(CreateTable(t).compile(self.database_connection))
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index e948b929..853d5d9f 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -101,10 +101,11 @@ def main():
         False,
         None,
         True,
-        True,
+        False,
         stix2.Directory,
     )
-    if store.database_exists:
+
+    if store.sink.database_exists:
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 

From 8cf2440ae67e93e0536d4951ec93258258df69b6 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:23:07 -0400
Subject: [PATCH 071/132] adding tests for process

---
 stix2/test/v21/test_process.py | 55 ++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 stix2/test/v21/test_process.py

diff --git a/stix2/test/v21/test_process.py b/stix2/test/v21/test_process.py
new file mode 100644
index 00000000..652597c0
--- /dev/null
+++ b/stix2/test/v21/test_process.py
@@ -0,0 +1,55 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+# errors when adding magic_number_hex to store, so ignoring for now
+
+process_dict = {
+    "type": "process",
+    "spec_version": "2.1",
+    "id": "process--f52a906a-0dfc-40bd-92f1-e7778ead38a9",
+    "is_hidden": False,
+    "pid": 1221,
+    "created_time": "2016-01-20T14:11:25.55Z",
+    "cwd": "/tmp/",
+    "environment_variables": {
+        "ENVTEST": "/path/to/bin"
+    },
+    "command_line": "./gedit-bin --new-window",
+    "opened_connection_refs": [
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3"
+    ],
+    "creator_user_ref": "user-account--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
+    "image_ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20", 
+    "parent_ref": "process--f52a906a-1dfc-40bd-92f1-e7778ead38a9", 
+    "child_refs": [
+        "process--ff2a906a-1dfc-40bd-92f1-e7778ead38a9",
+        "process--fe2a906a-1dfc-40bd-92f1-e7778ead38a9"
+    ]
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_process():
+    store.sink.generate_stix_schema()
+    process_stix_object = stix2.parse(process_dict)
+    store.add(process_stix_object)
+    read_obj = store.get(process_stix_object['id'])
+    read_obj = json.loads(store.get(process_stix_object['id']).serialize())
+
+    for attrib in process_dict.keys():
+        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables": # join multiple tables not implemented yet
+            continue
+        if attrib == "created_time":
+            assert stix2.utils.parse_into_datetime(process_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert process_dict[attrib] == read_obj[attrib]
+

From f3d54f2a71a04e38f03a3ae0af66589b07e573ac Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:26:16 -0400
Subject: [PATCH 072/132] adding test for software, removing incorrect comment
 in process

---
 stix2/test/v21/test_process.py  |  2 --
 stix2/test/v21/test_software.py | 34 +++++++++++++++++++++++++++++++++
 2 files changed, 34 insertions(+), 2 deletions(-)
 create mode 100644 stix2/test/v21/test_software.py

diff --git a/stix2/test/v21/test_process.py b/stix2/test/v21/test_process.py
index 652597c0..470cb836 100644
--- a/stix2/test/v21/test_process.py
+++ b/stix2/test/v21/test_process.py
@@ -5,8 +5,6 @@
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
-# errors when adding magic_number_hex to store, so ignoring for now
-
 process_dict = {
     "type": "process",
     "spec_version": "2.1",
diff --git a/stix2/test/v21/test_software.py b/stix2/test/v21/test_software.py
new file mode 100644
index 00000000..6357d2fc
--- /dev/null
+++ b/stix2/test/v21/test_software.py
@@ -0,0 +1,34 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+software_dict = {
+    "type": "software",
+    "spec_version": "2.1",
+    "id": "software--a1827f6d-ca53-5605-9e93-4316cd22a00a",
+    "name": "Word",
+    "cpe": "cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*", 
+    "version": "2002",
+    "vendor": "Microsoft"
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
+def test_software():
+    store.sink.generate_stix_schema()
+    software_stix_object = stix2.parse(software_dict)
+    store.add(software_stix_object)
+    read_obj = store.get(software_stix_object['id'])
+    read_obj = json.loads(store.get(software_stix_object['id']).serialize())
+
+    for attrib in software_dict.keys():
+        assert software_dict[attrib] == read_obj[attrib]
+

From bdf0ba7987d1b61ea8eadac3615b7754b21fc726 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:32:15 -0400
Subject: [PATCH 073/132] adding user_account test file

---
 stix2/test/v21/test_user_account.py | 49 +++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
 create mode 100644 stix2/test/v21/test_user_account.py

diff --git a/stix2/test/v21/test_user_account.py b/stix2/test/v21/test_user_account.py
new file mode 100644
index 00000000..f53625ec
--- /dev/null
+++ b/stix2/test/v21/test_user_account.py
@@ -0,0 +1,49 @@
+import datetime as dt
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+user_account_dict = {
+    "type": "user-account",
+    "spec_version": "2.1",
+    "id": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
+    "user_id": "1001",
+    "credential": "password",
+    "account_login": "jdoe",
+    "account_type": "unix",
+    "display_name": "John Doe",
+    "is_service_account": False,
+    "is_privileged": False,
+    "can_escalate_privs": True,
+    "is_disabled": False,
+    "account_created": "2016-01-20T12:31:12Z",
+    "account_expires": "2018-01-20T12:31:12Z",
+    "credential_last_changed": "2016-01-20T14:27:43Z",
+    "account_first_login": "2016-01-20T14:26:07Z",
+    "account_last_login": "2016-07-22T16:08:28Z"
+}
+
+store = RelationalDBStore(
+        "postgresql://postgres:admin@localhost/postgres",
+        False,
+        None,
+        True
+)
+
+def test_user_account():
+    store.sink.generate_stix_schema()
+    user_account_stix_object = stix2.parse(user_account_dict)
+    store.add(user_account_stix_object)
+    read_obj = json.loads(store.get(user_account_stix_object['id']).serialize())
+
+    for attrib in user_account_dict.keys():
+        if attrib == "account_created" or attrib == "account_expires" \
+            or attrib == "credential_last_changed" or attrib == "account_first_login" \
+                or attrib == "account_last_login":
+                    assert stix2.utils.parse_into_datetime(user_account_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+                    continue
+        assert user_account_dict[attrib] == read_obj[attrib]
+

From 0faa69ee546ccf12115c2baf3959283a0ba2c6f3 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:37:25 -0400
Subject: [PATCH 074/132] add test file for windows registry

---
 stix2/test/v21/test_windows_registry.py | 51 +++++++++++++++++++++++++
 1 file changed, 51 insertions(+)
 create mode 100644 stix2/test/v21/test_windows_registry.py

diff --git a/stix2/test/v21/test_windows_registry.py b/stix2/test/v21/test_windows_registry.py
new file mode 100644
index 00000000..68a275f1
--- /dev/null
+++ b/stix2/test/v21/test_windows_registry.py
@@ -0,0 +1,51 @@
+import datetime as dt
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+windows_registry_dict = {
+    "type": "windows-registry-key",
+    "spec_version": "2.1",
+    "id": "windows-registry-key--2ba37ae7-2745-5082-9dfd-9486dad41016",
+    "key": "hkey_local_machine\\system\\bar\\foo",
+    "values": [
+        {
+        "name": "Foo",
+        "data": "qwerty",
+        "data_type": "REG_SZ"
+        },
+        {
+        "name": "Bar",
+        "data": "42",
+        "data_type": "REG_DWORD"
+        }
+    ],
+    "modified_time": "2018-01-20T12:31:12Z",
+    "creator_user_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
+    "number_of_subkeys": 2
+}
+
+store = RelationalDBStore(
+        "postgresql://postgres:admin@localhost/postgres",
+        False,
+        None,
+        True
+)
+
+def test_windows_registry():
+    store.sink.generate_stix_schema()
+    windows_registry_stix_object = stix2.parse(windows_registry_dict)
+    store.add(windows_registry_stix_object)
+    read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
+
+    for attrib in windows_registry_dict.keys():
+        if attrib == "values": # skip multiple table join
+            continue
+        if attrib == "modified_time":
+            assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert windows_registry_dict[attrib] == read_obj[attrib]
+

From b7bd5ee851165c81fbbe6cb089d4c8c8ab023bc4 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 14:48:19 -0400
Subject: [PATCH 075/132] adding test file for x.509 certificates

---
 stix2/test/v21/test_x509_certificates.py | 83 ++++++++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 stix2/test/v21/test_x509_certificates.py

diff --git a/stix2/test/v21/test_x509_certificates.py b/stix2/test/v21/test_x509_certificates.py
new file mode 100644
index 00000000..580b8fc3
--- /dev/null
+++ b/stix2/test/v21/test_x509_certificates.py
@@ -0,0 +1,83 @@
+import pytest
+import stix2
+import json
+
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+basic_x509_certificate_dict = {
+    "type": "x509-certificate",
+    "spec_version": "2.1",
+    "id": "x509-certificate--463d7b2a-8516-5a50-a3d7-6f801465d5de",
+    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification  \
+    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
+    "validity_not_before": "2016-03-12T12:00:00Z",
+    "validity_not_after": "2016-08-21T12:00:00Z",
+    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
+    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
+    "serial_number": "36:f7:d4:32:f4:ab:70:ea:d3:ce:98:6e:ea:99:93:49:32:0a:b7:06"
+}
+
+extensions_x509_certificate_dict = {
+    "type":"x509-certificate",
+    "spec_version": "2.1",
+    "id": "x509-certificate--b595eaf0-0b28-5dad-9e8e-0fab9c1facc9",
+    "issuer":"C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification \
+    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
+    "validity_not_before":"2016-03-12T12:00:00Z",
+    "validity_not_after":"2016-08-21T12:00:00Z",
+    "subject":"C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
+    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
+    "serial_number": "02:08:87:83:f2:13:58:1f:79:52:1e:66:90:0a:02:24:c9:6b:c7:dc",
+    "x509_v3_extensions":{
+        "basic_constraints":"critical,CA:TRUE, pathlen:0",
+        "name_constraints":"permitted;IP:192.168.0.0/255.255.0.0",
+        "policy_constraints":"requireExplicitPolicy:3",
+        "key_usage":"critical, keyCertSign",
+        "extended_key_usage":"critical,codeSigning,1.2.3.4",
+        "subject_key_identifier":"hash",
+        "authority_key_identifier":"keyid,issuer",
+        "subject_alternative_name":"email:my@other.address,RID:1.2.3.4",
+        "issuer_alternative_name":"issuer:copy",
+        "crl_distribution_points":"URI:http://myhost.com/myca.crl",
+        "inhibit_any_policy":"2",
+        "private_key_usage_period_not_before":"2016-03-12T12:00:00Z",
+        "private_key_usage_period_not_after":"2018-03-12T12:00:00Z",
+        "certificate_policies":"1.2.4.5, 1.1.3.4"
+    }
+}
+
+store = RelationalDBStore(
+        "postgresql://postgres:admin@localhost/postgres",
+        False,
+        None,
+        True
+)
+
+def test_basic_x509_certificate():
+    store.sink.generate_stix_schema()
+    basic_x509_certificate_stix_object = stix2.parse(basic_x509_certificate_dict)
+    store.add(basic_x509_certificate_stix_object)
+    read_obj = json.loads(store.get(basic_x509_certificate_stix_object['id']).serialize())
+
+    for attrib in basic_x509_certificate_dict.keys():
+        if attrib == "validity_not_before" or attrib == "validity_not_after":
+            assert stix2.utils.parse_into_datetime(basic_x509_certificate_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert basic_x509_certificate_dict[attrib] == read_obj[attrib]
+    
+def test_x509_certificate_with_extensions():
+    store.sink.generate_stix_schema()
+    extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
+    store.add(extensions_x509_certificate_stix_object)
+    read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())
+
+    for attrib in extensions_x509_certificate_dict.keys():
+        if attrib == "x509_v3_extensions": # skipping multi-table join
+            continue
+        if attrib == "validity_not_before" or attrib == "validity_not_after":
+            assert stix2.utils.parse_into_datetime(extensions_x509_certificate_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert extensions_x509_certificate_dict[attrib] == read_obj[attrib]
+
+

From 21d35e4ee72c5e3ec64d81c94894097d55c09ada Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 15:02:54 -0400
Subject: [PATCH 076/132] adding sqlalchemy to setup.py

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index 80edabb4..79a40540 100644
--- a/setup.py
+++ b/setup.py
@@ -52,6 +52,7 @@ def get_long_description():
         'requests',
         'simplejson',
         'stix2-patterns>=1.2.0',
+        'sqlalchemy'
     ],
     project_urls={
         'Documentation': 'https://stix2.readthedocs.io/',

From ae3190fea610ea2e8d7f65d7e47d0b4a5465a206 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 15:07:59 -0400
Subject: [PATCH 077/132] adding inflection to setup.py requirements

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 79a40540..ff3dae87 100644
--- a/setup.py
+++ b/setup.py
@@ -52,7 +52,8 @@ def get_long_description():
         'requests',
         'simplejson',
         'stix2-patterns>=1.2.0',
-        'sqlalchemy'
+        'sqlalchemy',
+        'inflection'
     ],
     project_urls={
         'Documentation': 'https://stix2.readthedocs.io/',

From fa652d06ad0bad4d8eb76d90591abb1197027362 Mon Sep 17 00:00:00 2001
From: Gage Hackford <ghackford@mitre.org>
Date: Mon, 15 Apr 2024 15:15:47 -0400
Subject: [PATCH 078/132] adding psycopg2 to setup.py requirements

---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index ff3dae87..3d8d5813 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,8 @@ def get_long_description():
         'simplejson',
         'stix2-patterns>=1.2.0',
         'sqlalchemy',
-        'inflection'
+        'inflection',
+        'psycopg2'
     ],
     project_urls={
         'Documentation': 'https://stix2.readthedocs.io/',

From d2ecc73cd64f32b698c1ef7cf8891f659528be1e Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 09:23:47 -0400
Subject: [PATCH 079/132] added unique constraint

---
 stix2/datastore/relational_db/table_creation.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 6711fda3..3ad89a0a 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -2,7 +2,7 @@
 
 from sqlalchemy import (  # create_engine,; insert,
     ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text,
+    Integer, LargeBinary, Table, Text, UniqueConstraint,
 )
 
 from stix2.datastore.relational_db.add_method import add_method
@@ -99,7 +99,12 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
             nullable=False,
         ),
     )
-    return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
+    return Table(
+        canonicalize_table_name(table_name + "_" + name),
+        metadata,
+        *columns,
+        UniqueConstraint("id", "hash_name"),
+        schema=schema_name)
 
 
 def create_kill_chain_phases_table(name, metadata, schema_name, table_name):
@@ -366,7 +371,13 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
                     sql_type,
                 ),
             )
-    return [Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)]
+    return [
+        Table(canonicalize_table_name(table_name + "_" + name),
+              metadata,
+              *columns,
+              UniqueConstraint("id", "name"),
+              schema=schema_name)
+        ]
 
 
 @add_method(EmbeddedObjectProperty)

From 29550249cf587a3ffb1315f6b6fd504b930c0441 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 10:12:12 -0400
Subject: [PATCH 080/132] flaky

---
 setup.py                                      |  6 ++--
 .../datastore/relational_db/input_creation.py |  6 ++--
 .../datastore/relational_db/relational_db.py  |  4 +--
 .../datastore/relational_db/table_creation.py | 17 ++++++----
 stix2/test/v21/test_artifact.py               | 13 ++++----
 stix2/test/v21/test_autonomous_system.py      |  9 ++---
 stix2/test/v21/test_directory.py              | 11 ++++---
 stix2/test/v21/test_domain_name.py            |  9 ++---
 stix2/test/v21/test_email_address.py          |  9 ++---
 stix2/test/v21/test_email_message.py          | 33 ++++++++++---------
 stix2/test/v21/test_file.py                   | 13 ++++----
 stix2/test/v21/test_ipv4_ipv6.py              | 13 ++++----
 stix2/test/v21/test_mutex.py                  |  9 ++---
 stix2/test/v21/test_network_traffic.py        | 17 +++++-----
 stix2/test/v21/test_process.py                | 19 ++++++-----
 stix2/test/v21/test_software.py               | 11 ++++---
 stix2/test/v21/test_url.py                    |  9 ++---
 stix2/test/v21/test_user_account.py           |  9 ++---
 stix2/test/v21/test_windows_registry.py       | 15 +++++----
 stix2/test/v21/test_x509_certificates.py      | 15 +++++----
 tox.ini                                       |  4 +++
 21 files changed, 139 insertions(+), 112 deletions(-)

diff --git a/setup.py b/setup.py
index 3d8d5813..87911cb1 100644
--- a/setup.py
+++ b/setup.py
@@ -52,9 +52,7 @@ def get_long_description():
         'requests',
         'simplejson',
         'stix2-patterns>=1.2.0',
-        'sqlalchemy',
         'inflection',
-        'psycopg2'
     ],
     project_urls={
         'Documentation': 'https://stix2.readthedocs.io/',
@@ -64,5 +62,9 @@ def get_long_description():
     extras_require={
         'taxii': ['taxii2-client>=2.3.0'],
         'semantic': ['haversine', 'rapidfuzz'],
+        'relationaldb': [ 'sqlalchemy',
+                          'sqlalchemy_utils',
+                          'psycopg2',
+        ],
     },
 )
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 4e45e405..ae2b0742 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -158,8 +158,10 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(ListProperty)
-def generate_insert_information(self, name, stix_object, level=0, is_extension=False,
-                                foreign_key_value=None, schema_name=None, **kwargs):  # noqa: F811
+def generate_insert_information(
+    self, name, stix_object, level=0, is_extension=False,
+    foreign_key_value=None, schema_name=None, **kwargs,
+):  # noqa: F811
     data_sink = kwargs.get("data_sink")
     table_name = kwargs.get("table_name")
     if isinstance(self.contained, ReferenceProperty):
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 0fa674f5..6f23984c 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,6 +1,6 @@
-from sqlalchemy import MetaData, create_engine, select, delete
+from sqlalchemy import MetaData, create_engine, delete, select
 from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
-from sqlalchemy_utils import database_exists, create_database, drop_database
+from sqlalchemy_utils import create_database, database_exists, drop_database
 
 from stix2.base import _STIXBase
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 3ad89a0a..ee84e138 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -104,7 +104,8 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
         metadata,
         *columns,
         UniqueConstraint("id", "hash_name"),
-        schema=schema_name)
+        schema=schema_name,
+    )
 
 
 def create_kill_chain_phases_table(name, metadata, schema_name, table_name):
@@ -372,12 +373,14 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
                 ),
             )
     return [
-        Table(canonicalize_table_name(table_name + "_" + name),
-              metadata,
-              *columns,
-              UniqueConstraint("id", "name"),
-              schema=schema_name)
-        ]
+        Table(
+            canonicalize_table_name(table_name + "_" + name),
+            metadata,
+            *columns,
+            UniqueConstraint("id", "name"),
+            schema=schema_name,
+        ),
+    ]
 
 
 @add_method(EmbeddedObjectProperty)
diff --git a/stix2/test/v21/test_artifact.py b/stix2/test/v21/test_artifact.py
index 9c231a29..5c0e2eb4 100644
--- a/stix2/test/v21/test_artifact.py
+++ b/stix2/test/v21/test_artifact.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -10,7 +11,7 @@
     "spec_version": "2.1",
     "id": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
     "mime_type": "image/jpeg",
-    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh"
+    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh",
 }
 
 encrypted_artifact_dict = {
@@ -21,17 +22,17 @@
     "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhbiB1bnNhZmUgbWFsd2FyZSBiaW5hcnkh",
     "hashes": {
         "MD5": "6b885a1e1d42c0ca66e5f8a17e5a5d29",
-        "SHA-256": "3eea3c4819e9d387ff6809f13dde5426b9466285b7d923016b2842a13eb2888b"
+        "SHA-256": "3eea3c4819e9d387ff6809f13dde5426b9466285b7d923016b2842a13eb2888b",
     },
     "encryption_algorithm": "mime-type-indicated",
-    "decryption_key": "My voice is my passport"
+    "decryption_key": "My voice is my passport",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_basic_artifact():
diff --git a/stix2/test/v21/test_autonomous_system.py b/stix2/test/v21/test_autonomous_system.py
index aa4b5465..81399446 100644
--- a/stix2/test/v21/test_autonomous_system.py
+++ b/stix2/test/v21/test_autonomous_system.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
 as_dict = {
@@ -10,14 +11,14 @@
     "id": "autonomous-system--f822c34b-98ae-597f-ade5-27dc241e8c74",
     "number": 15139,
     "name": "Slime Industries",
-    "rir": "ARIN"
+    "rir": "ARIN",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_autonomous_system():
diff --git a/stix2/test/v21/test_directory.py b/stix2/test/v21/test_directory.py
index 94ca93f9..5896f25d 100644
--- a/stix2/test/v21/test_directory.py
+++ b/stix2/test/v21/test_directory.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
 directory_dict = {
@@ -14,15 +15,15 @@
     "path_enc": "cGF0aF9lbmM",
     "contains_refs": [
         "directory--94c0a9b0-520d-545d-9094-1a08ddf46b05",
-        "file--95c0a9b0-520d-545d-9094-1a08ddf46b05"
-    ]
+        "file--95c0a9b0-520d-545d-9094-1a08ddf46b05",
+    ],
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_directory():
diff --git a/stix2/test/v21/test_domain_name.py b/stix2/test/v21/test_domain_name.py
index e57a0625..42c60ae2 100644
--- a/stix2/test/v21/test_domain_name.py
+++ b/stix2/test/v21/test_domain_name.py
@@ -1,10 +1,11 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
-domain_name_dict = {    
+domain_name_dict = {
     "type": "domain-name",
     "spec_version": "2.1",
     "id": "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5",
@@ -15,7 +16,7 @@
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_autonomous_system():
diff --git a/stix2/test/v21/test_email_address.py b/stix2/test/v21/test_email_address.py
index 8ec4782c..a0992979 100644
--- a/stix2/test/v21/test_email_address.py
+++ b/stix2/test/v21/test_email_address.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -11,14 +12,14 @@
     "id": "email-addr--2d77a846-6264-5d51-b586-e43822ea1ea3",
     "value": "john@example.com",
     "display_name": "John Doe",
-    "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c"
+    "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_email_addr():
diff --git a/stix2/test/v21/test_email_message.py b/stix2/test/v21/test_email_message.py
index aef25458..7fad0946 100644
--- a/stix2/test/v21/test_email_message.py
+++ b/stix2/test/v21/test_email_message.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -14,11 +15,11 @@
     "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
     "cc_refs": [
         "email-addr--d2b3bf0c-f02a-51a1-8102-11aba7959868",
-        "email-addr--d3b3bf0c-f02a-51a1-8102-11aba7959868"
+        "email-addr--d3b3bf0c-f02a-51a1-8102-11aba7959868",
     ],
     "bcc_refs": [
         "email-addr--d4b3bf0c-f02a-51a1-8102-11aba7959868",
-        "email-addr--d5b3bf0c-f02a-51a1-8102-11aba7959868"
+        "email-addr--d5b3bf0c-f02a-51a1-8102-11aba7959868",
     ],
     "message_id": "message01",
     "is_multipart": False,
@@ -27,16 +28,16 @@
     "received_lines": [
         "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
         q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
-        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)"
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
     ],
     "additional_header_fields": {
         "Reply-To": [
             "steve@example.com",
-            "jane@example.com"
-        ]
+            "jane@example.com",
+        ],
     },
     "body": "message body",
-    "raw_email_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee"
+    "raw_email_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
 }
 
 multipart_email_msg_dict = {
@@ -47,7 +48,7 @@
     "received_lines": [
         "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
         q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
-        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)"
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
     ],
     "content_type": "multipart/mixed",
     "date": "2016-06-19T14:20:40.000Z",
@@ -58,32 +59,32 @@
     "additional_header_fields": {
         "Content-Disposition": "inline",
         "X-Mailer": "Mutt/1.5.23",
-        "X-Originating-IP": "198.51.100.3"
+        "X-Originating-IP": "198.51.100.3",
     },
     "body_multipart": [
         {
         "content_type": "text/plain; charset=utf-8",
         "content_disposition": "inline",
-        "body": "Cats are funny!"
+        "body": "Cats are funny!",
         },
         {
         "content_type": "image/png",
         "content_disposition": "attachment; filename=\"tabby.png\"",
-        "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5"
+        "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5",
         },
         {
         "content_type": "application/zip",
         "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
-        "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5"
-        }
-    ]
+        "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5",
+        },
+    ],
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_email_msg():
diff --git a/stix2/test/v21/test_file.py b/stix2/test/v21/test_file.py
index 32430858..12c1d5ee 100644
--- a/stix2/test/v21/test_file.py
+++ b/stix2/test/v21/test_file.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -12,7 +13,7 @@
     "spec_version": "2.1",
     "id": "file--66156fad-2a7d-5237-bbb4-ba1912887cfe",
     "hashes": {
-        "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a"
+        "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a",
     },
     "parent_directory_ref": "directory--93c0a9b0-520d-545d-9094-1a08ddf46b05",
     "name": "qwerty.dll",
@@ -23,16 +24,16 @@
     "mtime": "2018-11-23T08:17:27.000Z",
     "atime": "2018-11-23T08:17:27.000Z",
     "contains_refs": [
-        "file--77156fad-2a0d-5237-bba4-ba1912887cfe"
+        "file--77156fad-2a0d-5237-bba4-ba1912887cfe",
     ],
-    "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee"
+    "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_file():
diff --git a/stix2/test/v21/test_ipv4_ipv6.py b/stix2/test/v21/test_ipv4_ipv6.py
index 73f45541..a3755b3b 100644
--- a/stix2/test/v21/test_ipv4_ipv6.py
+++ b/stix2/test/v21/test_ipv4_ipv6.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -9,21 +10,21 @@
     "type": "ipv4-addr",
     "spec_version": "2.1",
     "id": "ipv4-addr--ff26c255-6336-5bc5-b98d-13d6226742dd",
-    "value": "198.51.100.3"
+    "value": "198.51.100.3",
 }
 
-ipv6_dict = { 
+ipv6_dict = {
     "type": "ipv6-addr",
     "spec_version": "2.1",
     "id": "ipv6-addr--1e61d36c-a26c-53b7-a80f-2a00161c96b1",
-    "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334"
+    "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_ipv4():
diff --git a/stix2/test/v21/test_mutex.py b/stix2/test/v21/test_mutex.py
index 0ef4a082..e7df076b 100644
--- a/stix2/test/v21/test_mutex.py
+++ b/stix2/test/v21/test_mutex.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -9,14 +10,14 @@
     "type": "mutex",
     "spec_version": "2.1",
     "id": "mutex--fba44954-d4e4-5d3b-814c-2b17dd8de300",
-    "name": "__CLEANSWEEP__"
+    "name": "__CLEANSWEEP__",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_mutex():
diff --git a/stix2/test/v21/test_network_traffic.py b/stix2/test/v21/test_network_traffic.py
index 3ca1c32c..f24e594f 100644
--- a/stix2/test/v21/test_network_traffic.py
+++ b/stix2/test/v21/test_network_traffic.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -20,26 +21,26 @@
     "dst_port": 1000,
     "protocols": [
         "ipv4",
-        "tcp"
+        "tcp",
     ],
     "src_byte_count": 147600,
     "dst_byte_count": 147600,
     "src_packets": 100,
     "dst_packets": 100,
     "src_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
-    "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03", 
+    "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03",
     "encapsulates_refs" : [
         "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
-        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4"
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4",
     ],
-    "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5"
+    "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_network_traffic():
@@ -50,7 +51,7 @@ def test_network_traffic():
 
     for attrib in network_traffic_dict.keys():
         if attrib == "encapsulates_refs": # multiple table join not implemented
-            continue 
+            continue
         if attrib == "start" or attrib == "end":
             assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
diff --git a/stix2/test/v21/test_process.py b/stix2/test/v21/test_process.py
index 470cb836..c58f9c68 100644
--- a/stix2/test/v21/test_process.py
+++ b/stix2/test/v21/test_process.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -14,26 +15,26 @@
     "created_time": "2016-01-20T14:11:25.55Z",
     "cwd": "/tmp/",
     "environment_variables": {
-        "ENVTEST": "/path/to/bin"
+        "ENVTEST": "/path/to/bin",
     },
     "command_line": "./gedit-bin --new-window",
     "opened_connection_refs": [
-        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3"
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
     ],
     "creator_user_ref": "user-account--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
-    "image_ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20", 
-    "parent_ref": "process--f52a906a-1dfc-40bd-92f1-e7778ead38a9", 
+    "image_ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20",
+    "parent_ref": "process--f52a906a-1dfc-40bd-92f1-e7778ead38a9",
     "child_refs": [
         "process--ff2a906a-1dfc-40bd-92f1-e7778ead38a9",
-        "process--fe2a906a-1dfc-40bd-92f1-e7778ead38a9"
-    ]
+        "process--fe2a906a-1dfc-40bd-92f1-e7778ead38a9",
+    ],
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_process():
diff --git a/stix2/test/v21/test_software.py b/stix2/test/v21/test_software.py
index 6357d2fc..a2a201ac 100644
--- a/stix2/test/v21/test_software.py
+++ b/stix2/test/v21/test_software.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -10,16 +11,16 @@
     "spec_version": "2.1",
     "id": "software--a1827f6d-ca53-5605-9e93-4316cd22a00a",
     "name": "Word",
-    "cpe": "cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*", 
+    "cpe": "cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
     "version": "2002",
-    "vendor": "Microsoft"
+    "vendor": "Microsoft",
 }
 
 store = RelationalDBStore(
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 def test_software():
diff --git a/stix2/test/v21/test_url.py b/stix2/test/v21/test_url.py
index ffa6ab15..d3a331e1 100644
--- a/stix2/test/v21/test_url.py
+++ b/stix2/test/v21/test_url.py
@@ -1,22 +1,23 @@
 import datetime as dt
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
 url_dict = {
     "type": "url",
     "id": "url--a5477287-23ac-5971-a010-5c287877fa60",
-    "value" : "https://example.com/research/index.html"
+    "value" : "https://example.com/research/index.html",
 }
 
 store = RelationalDBStore(
         "postgresql://postgres:admin@localhost/postgres",
         False,
         None,
-        True
+        True,
 )
 
 def test_url():
diff --git a/stix2/test/v21/test_user_account.py b/stix2/test/v21/test_user_account.py
index f53625ec..80c2fd14 100644
--- a/stix2/test/v21/test_user_account.py
+++ b/stix2/test/v21/test_user_account.py
@@ -1,8 +1,9 @@
 import datetime as dt
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -23,14 +24,14 @@
     "account_expires": "2018-01-20T12:31:12Z",
     "credential_last_changed": "2016-01-20T14:27:43Z",
     "account_first_login": "2016-01-20T14:26:07Z",
-    "account_last_login": "2016-07-22T16:08:28Z"
+    "account_last_login": "2016-07-22T16:08:28Z",
 }
 
 store = RelationalDBStore(
         "postgresql://postgres:admin@localhost/postgres",
         False,
         None,
-        True
+        True,
 )
 
 def test_user_account():
diff --git a/stix2/test/v21/test_windows_registry.py b/stix2/test/v21/test_windows_registry.py
index 68a275f1..582a1ced 100644
--- a/stix2/test/v21/test_windows_registry.py
+++ b/stix2/test/v21/test_windows_registry.py
@@ -1,8 +1,9 @@
 import datetime as dt
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -15,24 +16,24 @@
         {
         "name": "Foo",
         "data": "qwerty",
-        "data_type": "REG_SZ"
+        "data_type": "REG_SZ",
         },
         {
         "name": "Bar",
         "data": "42",
-        "data_type": "REG_DWORD"
-        }
+        "data_type": "REG_DWORD",
+        },
     ],
     "modified_time": "2018-01-20T12:31:12Z",
     "creator_user_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
-    "number_of_subkeys": 2
+    "number_of_subkeys": 2,
 }
 
 store = RelationalDBStore(
         "postgresql://postgres:admin@localhost/postgres",
         False,
         None,
-        True
+        True,
 )
 
 def test_windows_registry():
diff --git a/stix2/test/v21/test_x509_certificates.py b/stix2/test/v21/test_x509_certificates.py
index 580b8fc3..d3ec5b7f 100644
--- a/stix2/test/v21/test_x509_certificates.py
+++ b/stix2/test/v21/test_x509_certificates.py
@@ -1,7 +1,8 @@
-import pytest
-import stix2
 import json
 
+import pytest
+
+import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
@@ -15,7 +16,7 @@
     "validity_not_after": "2016-08-21T12:00:00Z",
     "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
     CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
-    "serial_number": "36:f7:d4:32:f4:ab:70:ea:d3:ce:98:6e:ea:99:93:49:32:0a:b7:06"
+    "serial_number": "36:f7:d4:32:f4:ab:70:ea:d3:ce:98:6e:ea:99:93:49:32:0a:b7:06",
 }
 
 extensions_x509_certificate_dict = {
@@ -43,15 +44,15 @@
         "inhibit_any_policy":"2",
         "private_key_usage_period_not_before":"2016-03-12T12:00:00Z",
         "private_key_usage_period_not_after":"2018-03-12T12:00:00Z",
-        "certificate_policies":"1.2.4.5, 1.1.3.4"
-    }
+        "certificate_policies":"1.2.4.5, 1.1.3.4",
+    },
 }
 
 store = RelationalDBStore(
         "postgresql://postgres:admin@localhost/postgres",
         False,
         None,
-        True
+        True,
 )
 
 def test_basic_x509_certificate():
@@ -65,7 +66,7 @@ def test_basic_x509_certificate():
             assert stix2.utils.parse_into_datetime(basic_x509_certificate_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert basic_x509_certificate_dict[attrib] == read_obj[attrib]
-    
+
 def test_x509_certificate_with_extensions():
     store.sink.generate_stix_schema()
     extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
diff --git a/tox.ini b/tox.ini
index bc7bd5b6..c74a1230 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,6 +11,10 @@ deps =
   rapidfuzz
   haversine
   medallion
+  sqlalchemy
+  sqlalchemy_utils
+  psycopg2
+
 commands =
   python -m pytest --cov=stix2 stix2/test/ --cov-report term-missing -W ignore::stix2.exceptions.STIXDeprecationWarning
 

From 3c92d7c513427fd0c149953c6256eb7538e9a5a6 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 10:14:57 -0400
Subject: [PATCH 081/132] flaky 2

---
 setup.py                                        | 7 ++++---
 stix2/datastore/relational_db/input_creation.py | 4 ++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/setup.py b/setup.py
index 87911cb1..f8048f58 100644
--- a/setup.py
+++ b/setup.py
@@ -62,9 +62,10 @@ def get_long_description():
     extras_require={
         'taxii': ['taxii2-client>=2.3.0'],
         'semantic': ['haversine', 'rapidfuzz'],
-        'relationaldb': [ 'sqlalchemy',
-                          'sqlalchemy_utils',
-                          'psycopg2',
+        'relationaldb': [
+            'sqlalchemy',
+            'sqlalchemy_utils',
+            'psycopg2',
         ],
     },
 )
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index ae2b0742..ee0a0a3c 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -158,10 +158,10 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(ListProperty)
-def generate_insert_information(
+def generate_insert_information(   # noqa: F811
     self, name, stix_object, level=0, is_extension=False,
     foreign_key_value=None, schema_name=None, **kwargs,
-):  # noqa: F811
+):
     data_sink = kwargs.get("data_sink")
     table_name = kwargs.get("table_name")
     if isinstance(self.contained, ReferenceProperty):

From b8782e334f449c5baee8b42f7e547d1122cd48d7 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 10:41:04 -0400
Subject: [PATCH 082/132] flaky 3

---
 stix2/test/v21/test_artifact.py          |  8 +--
 stix2/test/v21/test_autonomous_system.py |  5 +-
 stix2/test/v21/test_directory.py         |  8 ++-
 stix2/test/v21/test_domain_name.py       |  5 +-
 stix2/test/v21/test_email_address.py     |  3 +-
 stix2/test/v21/test_email_message.py     | 35 +++++++------
 stix2/test/v21/test_file.py              |  6 +--
 stix2/test/v21/test_ipv4_ipv6.py         |  6 +--
 stix2/test/v21/test_mutex.py             |  6 +--
 stix2/test/v21/test_network_traffic.py   | 12 ++---
 stix2/test/v21/test_process.py           |  7 ++-
 stix2/test/v21/test_software.py          |  4 +-
 stix2/test/v21/test_url.py               |  8 ++-
 stix2/test/v21/test_user_account.py      | 21 ++++----
 stix2/test/v21/test_windows_registry.py  | 31 ++++++------
 stix2/test/v21/test_x509_certificates.py | 64 ++++++++++++------------
 16 files changed, 103 insertions(+), 126 deletions(-)

diff --git a/stix2/test/v21/test_artifact.py b/stix2/test/v21/test_artifact.py
index 5c0e2eb4..2c68c0d2 100644
--- a/stix2/test/v21/test_artifact.py
+++ b/stix2/test/v21/test_artifact.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -35,6 +33,7 @@
     True,
 )
 
+
 def test_basic_artifact():
     store.sink.generate_stix_schema()
     artifact_stix_object = stix2.parse(basic_artifact_dict)
@@ -44,6 +43,7 @@ def test_basic_artifact():
     for attrib in basic_artifact_dict.keys():
         assert basic_artifact_dict[attrib] == read_obj[attrib]
 
+
 def test_encrypted_artifact():
     store.sink.generate_stix_schema()
     artifact_stix_object = stix2.parse(encrypted_artifact_dict)
@@ -51,6 +51,6 @@ def test_encrypted_artifact():
     read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
 
     for attrib in encrypted_artifact_dict.keys():
-        if attrib == 'hashes': # TODO hashes are saved to separate table, functionality to retrieve is WIP
+        if attrib == 'hashes':  # TODO hashes are saved to separate table, functionality to retrieve is WIP
             continue
-        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
\ No newline at end of file
+        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_autonomous_system.py b/stix2/test/v21/test_autonomous_system.py
index 81399446..82fb4f10 100644
--- a/stix2/test/v21/test_autonomous_system.py
+++ b/stix2/test/v21/test_autonomous_system.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
@@ -21,6 +19,7 @@
     True,
 )
 
+
 def test_autonomous_system():
     store.sink.generate_stix_schema()
     as_obj = stix2.parse(as_dict)
@@ -28,4 +27,4 @@ def test_autonomous_system():
     read_obj = json.loads(store.get(as_obj['id']).serialize())
 
     for attrib in as_dict.keys():
-        assert as_dict[attrib] == read_obj[attrib]
\ No newline at end of file
+        assert as_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_directory.py b/stix2/test/v21/test_directory.py
index 5896f25d..2f413866 100644
--- a/stix2/test/v21/test_directory.py
+++ b/stix2/test/v21/test_directory.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
@@ -26,6 +24,7 @@
     True,
 )
 
+
 def test_directory():
     store.sink.generate_stix_schema()
     directory_obj = stix2.parse(directory_dict)
@@ -33,10 +32,9 @@ def test_directory():
     read_obj = json.loads(store.get(directory_obj['id']).serialize())
 
     for attrib in directory_dict.keys():
-        if attrib == "contains_refs": # TODO remove skip once we can pull from table join
+        if attrib == "contains_refs":  # TODO remove skip once we can pull from table join
             continue
-        if attrib == "ctime" or attrib == "mtime": # convert both into stix2 date format for consistency
+        if attrib == "ctime" or attrib == "mtime":  # convert both into stix2 date format for consistency
             assert stix2.utils.parse_into_datetime(directory_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert directory_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_domain_name.py b/stix2/test/v21/test_domain_name.py
index 42c60ae2..93f30d54 100644
--- a/stix2/test/v21/test_domain_name.py
+++ b/stix2/test/v21/test_domain_name.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 
@@ -19,6 +17,7 @@
     True,
 )
 
+
 def test_autonomous_system():
     store.sink.generate_stix_schema()
     domain_name_obj = stix2.parse(domain_name_dict)
@@ -26,4 +25,4 @@ def test_autonomous_system():
     read_obj = json.loads(store.get(domain_name_obj['id']).serialize())
 
     for attrib in domain_name_dict.keys():
-        assert domain_name_dict[attrib] == read_obj[attrib]
\ No newline at end of file
+        assert domain_name_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_email_address.py b/stix2/test/v21/test_email_address.py
index a0992979..6a00daef 100644
--- a/stix2/test/v21/test_email_address.py
+++ b/stix2/test/v21/test_email_address.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -22,6 +20,7 @@
     True,
 )
 
+
 def test_email_addr():
     store.sink.generate_stix_schema()
     email_addr_stix_object = stix2.parse(email_addr_dict)
diff --git a/stix2/test/v21/test_email_message.py b/stix2/test/v21/test_email_message.py
index 7fad0946..038b3274 100644
--- a/stix2/test/v21/test_email_message.py
+++ b/stix2/test/v21/test_email_message.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -63,19 +61,19 @@
     },
     "body_multipart": [
         {
-        "content_type": "text/plain; charset=utf-8",
-        "content_disposition": "inline",
-        "body": "Cats are funny!",
+            "content_type": "text/plain; charset=utf-8",
+            "content_disposition": "inline",
+            "body": "Cats are funny!",
         },
         {
-        "content_type": "image/png",
-        "content_disposition": "attachment; filename=\"tabby.png\"",
-        "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5",
+            "content_type": "image/png",
+            "content_disposition": "attachment; filename=\"tabby.png\"",
+            "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5",
         },
         {
-        "content_type": "application/zip",
-        "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
-        "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5",
+            "content_type": "application/zip",
+            "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
+            "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5",
         },
     ],
 }
@@ -87,6 +85,7 @@
     True,
 )
 
+
 def test_email_msg():
     store.sink.generate_stix_schema()
     email_msg_stix_object = stix2.parse(email_msg_dict)
@@ -95,13 +94,16 @@ def test_email_msg():
 
     for attrib in email_msg_dict.keys():
         if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-            or attrib == "additional_header_fields": # join multiple tables not implemented yet
+                or attrib == "additional_header_fields":  # join multiple tables not implemented yet
             continue
         if attrib == "date":
-            assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
             continue
         assert email_msg_dict[attrib] == read_obj[attrib]
 
+
 def test_multipart_email_msg():
     store.sink.generate_stix_schema()
     multipart_email_msg_stix_object = stix2.parse(multipart_email_msg_dict)
@@ -110,10 +112,11 @@ def test_multipart_email_msg():
 
     for attrib in multipart_email_msg_dict.keys():
         if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-            or attrib == "additional_header_fields" or attrib == "body_multipart": # join multiple tables not implemented yet
+                or attrib == "additional_header_fields" or attrib == "body_multipart":  # join multiple tables not implemented yet
             continue
         if attrib == "date":
-            assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
             continue
         assert multipart_email_msg_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_file.py b/stix2/test/v21/test_file.py
index 12c1d5ee..647f81e5 100644
--- a/stix2/test/v21/test_file.py
+++ b/stix2/test/v21/test_file.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -36,6 +34,7 @@
     True,
 )
 
+
 def test_file():
     store.sink.generate_stix_schema()
     file_stix_object = stix2.parse(file_dict)
@@ -44,10 +43,9 @@ def test_file():
     read_obj = json.loads(store.get(file_stix_object['id']).serialize())
 
     for attrib in file_dict.keys():
-        if attrib == "contains_refs" or attrib == "hashes": # join multiple tables not implemented yet
+        if attrib == "contains_refs" or attrib == "hashes":  # join multiple tables not implemented yet
             continue
         if attrib == "ctime" or attrib == "mtime" or attrib == "atime":
             assert stix2.utils.parse_into_datetime(file_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert file_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_ipv4_ipv6.py b/stix2/test/v21/test_ipv4_ipv6.py
index a3755b3b..c32197d4 100644
--- a/stix2/test/v21/test_ipv4_ipv6.py
+++ b/stix2/test/v21/test_ipv4_ipv6.py
@@ -1,7 +1,3 @@
-import json
-
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -27,6 +23,7 @@
     True,
 )
 
+
 def test_ipv4():
     store.sink.generate_stix_schema()
     ipv4_stix_object = stix2.parse(ipv4_dict)
@@ -45,4 +42,3 @@ def test_ipv6():
 
     for attrib in ipv6_dict.keys():
         assert ipv6_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_mutex.py b/stix2/test/v21/test_mutex.py
index e7df076b..55fdd5d2 100644
--- a/stix2/test/v21/test_mutex.py
+++ b/stix2/test/v21/test_mutex.py
@@ -1,7 +1,3 @@
-import json
-
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -20,6 +16,7 @@
     True,
 )
 
+
 def test_mutex():
     store.sink.generate_stix_schema()
     mutex_stix_object = stix2.parse(mutex_dict)
@@ -28,4 +25,3 @@ def test_mutex():
 
     for attrib in mutex_dict.keys():
         assert mutex_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_network_traffic.py b/stix2/test/v21/test_network_traffic.py
index f24e594f..ddb47a49 100644
--- a/stix2/test/v21/test_network_traffic.py
+++ b/stix2/test/v21/test_network_traffic.py
@@ -1,7 +1,3 @@
-import json
-
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -29,7 +25,7 @@
     "dst_packets": 100,
     "src_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
     "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03",
-    "encapsulates_refs" : [
+    "encapsulates_refs": [
         "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
         "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4",
     ],
@@ -43,6 +39,7 @@
     True,
 )
 
+
 def test_network_traffic():
     store.sink.generate_stix_schema()
     network_traffic_stix_object = stix2.parse(network_traffic_dict)
@@ -50,12 +47,9 @@ def test_network_traffic():
     read_obj = store.get(network_traffic_stix_object['id'])
 
     for attrib in network_traffic_dict.keys():
-        if attrib == "encapsulates_refs": # multiple table join not implemented
+        if attrib == "encapsulates_refs":  # multiple table join not implemented
             continue
         if attrib == "start" or attrib == "end":
             assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert network_traffic_dict[attrib] == read_obj[attrib]
-
-
-
diff --git a/stix2/test/v21/test_process.py b/stix2/test/v21/test_process.py
index c58f9c68..519fd2d1 100644
--- a/stix2/test/v21/test_process.py
+++ b/stix2/test/v21/test_process.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -37,6 +35,7 @@
     True,
 )
 
+
 def test_process():
     store.sink.generate_stix_schema()
     process_stix_object = stix2.parse(process_dict)
@@ -45,10 +44,10 @@ def test_process():
     read_obj = json.loads(store.get(process_stix_object['id']).serialize())
 
     for attrib in process_dict.keys():
-        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables": # join multiple tables not implemented yet
+        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables":
+            # join multiple tables not implemented yet
             continue
         if attrib == "created_time":
             assert stix2.utils.parse_into_datetime(process_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert process_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_software.py b/stix2/test/v21/test_software.py
index a2a201ac..896e9c2a 100644
--- a/stix2/test/v21/test_software.py
+++ b/stix2/test/v21/test_software.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -23,6 +21,7 @@
     True,
 )
 
+
 def test_software():
     store.sink.generate_stix_schema()
     software_stix_object = stix2.parse(software_dict)
@@ -32,4 +31,3 @@ def test_software():
 
     for attrib in software_dict.keys():
         assert software_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_url.py b/stix2/test/v21/test_url.py
index d3a331e1..838cbfbb 100644
--- a/stix2/test/v21/test_url.py
+++ b/stix2/test/v21/test_url.py
@@ -1,8 +1,5 @@
-import datetime as dt
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -10,7 +7,7 @@
 url_dict = {
     "type": "url",
     "id": "url--a5477287-23ac-5971-a010-5c287877fa60",
-    "value" : "https://example.com/research/index.html",
+    "value": "https://example.com/research/index.html",
 }
 
 store = RelationalDBStore(
@@ -20,6 +17,7 @@
         True,
 )
 
+
 def test_url():
     store.sink.generate_stix_schema()
     url_stix_object = stix2.parse(url_dict)
@@ -27,4 +25,4 @@ def test_url():
     read_obj = json.loads(store.get(url_stix_object['id']).serialize())
 
     for attrib in url_dict.keys():
-        assert url_dict[attrib] == read_obj[attrib]
\ No newline at end of file
+        assert url_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_user_account.py b/stix2/test/v21/test_user_account.py
index 80c2fd14..374c2377 100644
--- a/stix2/test/v21/test_user_account.py
+++ b/stix2/test/v21/test_user_account.py
@@ -1,8 +1,5 @@
-import datetime as dt
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -28,12 +25,13 @@
 }
 
 store = RelationalDBStore(
-        "postgresql://postgres:admin@localhost/postgres",
-        False,
-        None,
-        True,
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
 )
 
+
 def test_user_account():
     store.sink.generate_stix_schema()
     user_account_stix_object = stix2.parse(user_account_dict)
@@ -42,9 +40,10 @@ def test_user_account():
 
     for attrib in user_account_dict.keys():
         if attrib == "account_created" or attrib == "account_expires" \
-            or attrib == "credential_last_changed" or attrib == "account_first_login" \
+                or attrib == "credential_last_changed" or attrib == "account_first_login" \
                 or attrib == "account_last_login":
-                    assert stix2.utils.parse_into_datetime(user_account_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
-                    continue
+            assert stix2.utils.parse_into_datetime(user_account_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
+            continue
         assert user_account_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_windows_registry.py b/stix2/test/v21/test_windows_registry.py
index 582a1ced..f2864548 100644
--- a/stix2/test/v21/test_windows_registry.py
+++ b/stix2/test/v21/test_windows_registry.py
@@ -1,8 +1,5 @@
-import datetime as dt
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -14,14 +11,14 @@
     "key": "hkey_local_machine\\system\\bar\\foo",
     "values": [
         {
-        "name": "Foo",
-        "data": "qwerty",
-        "data_type": "REG_SZ",
+            "name": "Foo",
+            "data": "qwerty",
+            "data_type": "REG_SZ",
         },
         {
-        "name": "Bar",
-        "data": "42",
-        "data_type": "REG_DWORD",
+            "name": "Bar",
+            "data": "42",
+            "data_type": "REG_DWORD",
         },
     ],
     "modified_time": "2018-01-20T12:31:12Z",
@@ -30,12 +27,13 @@
 }
 
 store = RelationalDBStore(
-        "postgresql://postgres:admin@localhost/postgres",
-        False,
-        None,
-        True,
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
 )
 
+
 def test_windows_registry():
     store.sink.generate_stix_schema()
     windows_registry_stix_object = stix2.parse(windows_registry_dict)
@@ -43,10 +41,11 @@ def test_windows_registry():
     read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
 
     for attrib in windows_registry_dict.keys():
-        if attrib == "values": # skip multiple table join
+        if attrib == "values":  # skip multiple table join
             continue
         if attrib == "modified_time":
-            assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
             continue
         assert windows_registry_dict[attrib] == read_obj[attrib]
-
diff --git a/stix2/test/v21/test_x509_certificates.py b/stix2/test/v21/test_x509_certificates.py
index d3ec5b7f..1847dbc2 100644
--- a/stix2/test/v21/test_x509_certificates.py
+++ b/stix2/test/v21/test_x509_certificates.py
@@ -1,7 +1,5 @@
 import json
 
-import pytest
-
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
@@ -20,41 +18,42 @@
 }
 
 extensions_x509_certificate_dict = {
-    "type":"x509-certificate",
+    "type": "x509-certificate",
     "spec_version": "2.1",
     "id": "x509-certificate--b595eaf0-0b28-5dad-9e8e-0fab9c1facc9",
-    "issuer":"C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification \
+    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification \
     Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
-    "validity_not_before":"2016-03-12T12:00:00Z",
-    "validity_not_after":"2016-08-21T12:00:00Z",
-    "subject":"C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
+    "validity_not_before": "2016-03-12T12:00:00Z",
+    "validity_not_after": "2016-08-21T12:00:00Z",
+    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
     CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
     "serial_number": "02:08:87:83:f2:13:58:1f:79:52:1e:66:90:0a:02:24:c9:6b:c7:dc",
-    "x509_v3_extensions":{
-        "basic_constraints":"critical,CA:TRUE, pathlen:0",
-        "name_constraints":"permitted;IP:192.168.0.0/255.255.0.0",
-        "policy_constraints":"requireExplicitPolicy:3",
-        "key_usage":"critical, keyCertSign",
-        "extended_key_usage":"critical,codeSigning,1.2.3.4",
-        "subject_key_identifier":"hash",
-        "authority_key_identifier":"keyid,issuer",
-        "subject_alternative_name":"email:my@other.address,RID:1.2.3.4",
-        "issuer_alternative_name":"issuer:copy",
-        "crl_distribution_points":"URI:http://myhost.com/myca.crl",
-        "inhibit_any_policy":"2",
-        "private_key_usage_period_not_before":"2016-03-12T12:00:00Z",
-        "private_key_usage_period_not_after":"2018-03-12T12:00:00Z",
-        "certificate_policies":"1.2.4.5, 1.1.3.4",
+    "x509_v3_extensions": {
+        "basic_constraints": "critical,CA:TRUE, pathlen:0",
+        "name_constraints": "permitted;IP:192.168.0.0/255.255.0.0",
+        "policy_constraints": "requireExplicitPolicy:3",
+        "key_usage": "critical, keyCertSign",
+        "extended_key_usage": "critical,codeSigning,1.2.3.4",
+        "subject_key_identifier": "hash",
+        "authority_key_identifier": "keyid,issuer",
+        "subject_alternative_name": "email:my@other.address,RID:1.2.3.4",
+        "issuer_alternative_name": "issuer:copy",
+        "crl_distribution_points": "URI:http://myhost.com/myca.crl",
+        "inhibit_any_policy": "2",
+        "private_key_usage_period_not_before": "2016-03-12T12:00:00Z",
+        "private_key_usage_period_not_after": "2018-03-12T12:00:00Z",
+        "certificate_policies": "1.2.4.5, 1.1.3.4",
     },
 }
 
 store = RelationalDBStore(
-        "postgresql://postgres:admin@localhost/postgres",
-        False,
-        None,
-        True,
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
 )
 
+
 def test_basic_x509_certificate():
     store.sink.generate_stix_schema()
     basic_x509_certificate_stix_object = stix2.parse(basic_x509_certificate_dict)
@@ -63,10 +62,13 @@ def test_basic_x509_certificate():
 
     for attrib in basic_x509_certificate_dict.keys():
         if attrib == "validity_not_before" or attrib == "validity_not_after":
-            assert stix2.utils.parse_into_datetime(basic_x509_certificate_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            assert stix2.utils.parse_into_datetime(
+                basic_x509_certificate_dict[attrib],
+            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert basic_x509_certificate_dict[attrib] == read_obj[attrib]
 
+
 def test_x509_certificate_with_extensions():
     store.sink.generate_stix_schema()
     extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
@@ -74,11 +76,11 @@ def test_x509_certificate_with_extensions():
     read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())
 
     for attrib in extensions_x509_certificate_dict.keys():
-        if attrib == "x509_v3_extensions": # skipping multi-table join
+        if attrib == "x509_v3_extensions":  # skipping multi-table join
             continue
         if attrib == "validity_not_before" or attrib == "validity_not_after":
-            assert stix2.utils.parse_into_datetime(extensions_x509_certificate_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            assert stix2.utils.parse_into_datetime(
+                extensions_x509_certificate_dict[attrib],
+            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert extensions_x509_certificate_dict[attrib] == read_obj[attrib]
-
-

From a964b6c097404e57d1e958fef39b961183f3fad5 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Tue, 16 Apr 2024 11:59:22 -0400
Subject: [PATCH 083/132] Merge datastore relation test

---
 stix2/test/v21/test_artifact.py               |  56 --
 stix2/test/v21/test_autonomous_system.py      |  30 -
 .../test/v21/test_datastore_relational_db.py  | 748 ++++++++++++++++++
 stix2/test/v21/test_directory.py              |  40 -
 stix2/test/v21/test_domain_name.py            |  28 -
 stix2/test/v21/test_email_address.py          |  31 -
 stix2/test/v21/test_email_message.py          | 122 ---
 stix2/test/v21/test_file.py                   |  51 --
 stix2/test/v21/test_ipv4_ipv6.py              |  44 --
 stix2/test/v21/test_mutex.py                  |  27 -
 stix2/test/v21/test_network_traffic.py        |  55 --
 stix2/test/v21/test_process.py                |  53 --
 stix2/test/v21/test_software.py               |  33 -
 stix2/test/v21/test_url.py                    |  28 -
 stix2/test/v21/test_user_account.py           |  49 --
 stix2/test/v21/test_windows_registry.py       |  51 --
 stix2/test/v21/test_x509_certificates.py      |  86 --
 17 files changed, 748 insertions(+), 784 deletions(-)
 delete mode 100644 stix2/test/v21/test_artifact.py
 delete mode 100644 stix2/test/v21/test_autonomous_system.py
 create mode 100644 stix2/test/v21/test_datastore_relational_db.py
 delete mode 100644 stix2/test/v21/test_directory.py
 delete mode 100644 stix2/test/v21/test_domain_name.py
 delete mode 100644 stix2/test/v21/test_email_address.py
 delete mode 100644 stix2/test/v21/test_email_message.py
 delete mode 100644 stix2/test/v21/test_file.py
 delete mode 100644 stix2/test/v21/test_ipv4_ipv6.py
 delete mode 100644 stix2/test/v21/test_mutex.py
 delete mode 100644 stix2/test/v21/test_network_traffic.py
 delete mode 100644 stix2/test/v21/test_process.py
 delete mode 100644 stix2/test/v21/test_software.py
 delete mode 100644 stix2/test/v21/test_url.py
 delete mode 100644 stix2/test/v21/test_user_account.py
 delete mode 100644 stix2/test/v21/test_windows_registry.py
 delete mode 100644 stix2/test/v21/test_x509_certificates.py

diff --git a/stix2/test/v21/test_artifact.py b/stix2/test/v21/test_artifact.py
deleted file mode 100644
index 2c68c0d2..00000000
--- a/stix2/test/v21/test_artifact.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-basic_artifact_dict = {
-    "type": "artifact",
-    "spec_version": "2.1",
-    "id": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
-    "mime_type": "image/jpeg",
-    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh",
-}
-
-encrypted_artifact_dict = {
-    "type": "artifact",
-    "spec_version": "2.1",
-    "id": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
-    "mime_type": "application/zip",
-    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhbiB1bnNhZmUgbWFsd2FyZSBiaW5hcnkh",
-    "hashes": {
-        "MD5": "6b885a1e1d42c0ca66e5f8a17e5a5d29",
-        "SHA-256": "3eea3c4819e9d387ff6809f13dde5426b9466285b7d923016b2842a13eb2888b",
-    },
-    "encryption_algorithm": "mime-type-indicated",
-    "decryption_key": "My voice is my passport",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_basic_artifact():
-    store.sink.generate_stix_schema()
-    artifact_stix_object = stix2.parse(basic_artifact_dict)
-    store.add(artifact_stix_object)
-    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
-
-    for attrib in basic_artifact_dict.keys():
-        assert basic_artifact_dict[attrib] == read_obj[attrib]
-
-
-def test_encrypted_artifact():
-    store.sink.generate_stix_schema()
-    artifact_stix_object = stix2.parse(encrypted_artifact_dict)
-    store.add(artifact_stix_object)
-    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
-
-    for attrib in encrypted_artifact_dict.keys():
-        if attrib == 'hashes':  # TODO hashes are saved to separate table, functionality to retrieve is WIP
-            continue
-        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_autonomous_system.py b/stix2/test/v21/test_autonomous_system.py
deleted file mode 100644
index 82fb4f10..00000000
--- a/stix2/test/v21/test_autonomous_system.py
+++ /dev/null
@@ -1,30 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-
-as_dict = {
-    "type": "autonomous-system",
-    "spec_version": "2.1",
-    "id": "autonomous-system--f822c34b-98ae-597f-ade5-27dc241e8c74",
-    "number": 15139,
-    "name": "Slime Industries",
-    "rir": "ARIN",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_autonomous_system():
-    store.sink.generate_stix_schema()
-    as_obj = stix2.parse(as_dict)
-    store.add(as_obj)
-    read_obj = json.loads(store.get(as_obj['id']).serialize())
-
-    for attrib in as_dict.keys():
-        assert as_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
new file mode 100644
index 00000000..746b3333
--- /dev/null
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -0,0 +1,748 @@
+import json
+
+import stix2
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
+import stix2.properties
+
+# Artifacts
+basic_artifact_dict = {
+    "type": "artifact",
+    "spec_version": "2.1",
+    "id": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
+    "mime_type": "image/jpeg",
+    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhIHNhZmUgbWFsd2FyZSBiaW5hcnkh",
+}
+
+encrypted_artifact_dict = {
+    "type": "artifact",
+    "spec_version": "2.1",
+    "id": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
+    "mime_type": "application/zip",
+    "payload_bin": "VGhpcyBpcyBhIHBsYWNlaG9sZGVyIGZvciBhbiB1bnNhZmUgbWFsd2FyZSBiaW5hcnkh",
+    "hashes": {
+        "MD5": "6b885a1e1d42c0ca66e5f8a17e5a5d29",
+        "SHA-256": "3eea3c4819e9d387ff6809f13dde5426b9466285b7d923016b2842a13eb2888b",
+    },
+    "encryption_algorithm": "mime-type-indicated",
+    "decryption_key": "My voice is my passport",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_basic_artifact():
+    store.sink.generate_stix_schema()
+    artifact_stix_object = stix2.parse(basic_artifact_dict)
+    store.add(artifact_stix_object)
+    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
+
+    for attrib in basic_artifact_dict.keys():
+        assert basic_artifact_dict[attrib] == read_obj[attrib]
+
+
+def test_encrypted_artifact():
+    store.sink.generate_stix_schema()
+    artifact_stix_object = stix2.parse(encrypted_artifact_dict)
+    store.add(artifact_stix_object)
+    read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
+
+    for attrib in encrypted_artifact_dict.keys():
+        if attrib == 'hashes':  # TODO hashes are saved to separate table, functionality to retrieve is WIP
+            continue
+        assert encrypted_artifact_dict[attrib] == read_obj[attrib]
+
+# Autonomous System
+as_dict = {
+    "type": "autonomous-system",
+    "spec_version": "2.1",
+    "id": "autonomous-system--f822c34b-98ae-597f-ade5-27dc241e8c74",
+    "number": 15139,
+    "name": "Slime Industries",
+    "rir": "ARIN",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_autonomous_system():
+    store.sink.generate_stix_schema()
+    as_obj = stix2.parse(as_dict)
+    store.add(as_obj)
+    read_obj = json.loads(store.get(as_obj['id']).serialize())
+
+    for attrib in as_dict.keys():
+        assert as_dict[attrib] == read_obj[attrib]
+
+# Directory
+directory_dict = {
+    "type": "directory",
+    "spec_version": "2.1",
+    "id": "directory--17c909b1-521d-545d-9094-1a08ddf46b05",
+    "ctime": "2018-11-23T08:17:27.000Z",
+    "mtime": "2018-11-23T08:17:27.000Z",
+    "path": "C:\\Windows\\System32",
+    "path_enc": "cGF0aF9lbmM",
+    "contains_refs": [
+        "directory--94c0a9b0-520d-545d-9094-1a08ddf46b05",
+        "file--95c0a9b0-520d-545d-9094-1a08ddf46b05",
+    ],
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_directory():
+    store.sink.generate_stix_schema()
+    directory_obj = stix2.parse(directory_dict)
+    store.add(directory_obj)
+    read_obj = json.loads(store.get(directory_obj['id']).serialize())
+
+    for attrib in directory_dict.keys():
+        if attrib == "contains_refs":  # TODO remove skip once we can pull from table join
+            continue
+        if attrib == "ctime" or attrib == "mtime":  # convert both into stix2 date format for consistency
+            assert stix2.utils.parse_into_datetime(directory_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert directory_dict[attrib] == read_obj[attrib]
+
+# Domain Name
+domain_name_dict = {
+    "type": "domain-name",
+    "spec_version": "2.1",
+    "id": "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5",
+    "value": "example.com",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_autonomous_system():
+    store.sink.generate_stix_schema()
+    domain_name_obj = stix2.parse(domain_name_dict)
+    store.add(domain_name_obj)
+    read_obj = json.loads(store.get(domain_name_obj['id']).serialize())
+
+    for attrib in domain_name_dict.keys():
+        assert domain_name_dict[attrib] == read_obj[attrib]
+
+# Email Address
+email_addr_dict = {
+    "type": "email-addr",
+    "spec_version": "2.1",
+    "id": "email-addr--2d77a846-6264-5d51-b586-e43822ea1ea3",
+    "value": "john@example.com",
+    "display_name": "John Doe",
+    "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_email_addr():
+    store.sink.generate_stix_schema()
+    email_addr_stix_object = stix2.parse(email_addr_dict)
+    store.add(email_addr_stix_object)
+    read_obj = json.loads(store.get(email_addr_stix_object['id']).serialize())
+
+    for attrib in email_addr_dict.keys():
+        assert email_addr_dict[attrib] == read_obj[attrib]
+
+# Email Message
+email_msg_dict = {
+    "type": "email-message",
+    "spec_version": "2.1",
+    "id": "email-message--8c57a381-2a17-5e61-8754-5ef96efb286c",
+    "from_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5dda",
+    "sender_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5eeb",
+    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
+    "cc_refs": [
+        "email-addr--d2b3bf0c-f02a-51a1-8102-11aba7959868",
+        "email-addr--d3b3bf0c-f02a-51a1-8102-11aba7959868",
+    ],
+    "bcc_refs": [
+        "email-addr--d4b3bf0c-f02a-51a1-8102-11aba7959868",
+        "email-addr--d5b3bf0c-f02a-51a1-8102-11aba7959868",
+    ],
+    "message_id": "message01",
+    "is_multipart": False,
+    "date": "2004-04-19T12:22:23.000Z",
+    "subject": "Did you see this?",
+    "received_lines": [
+        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
+        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
+    ],
+    "additional_header_fields": {
+        "Reply-To": [
+            "steve@example.com",
+            "jane@example.com",
+        ],
+    },
+    "body": "message body",
+    "raw_email_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
+}
+
+multipart_email_msg_dict = {
+    "type": "email-message",
+    "spec_version": "2.1",
+    "id": "email-message--ef9b4b7f-14c8-5955-8065-020e0316b559",
+    "is_multipart": True,
+    "received_lines": [
+        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
+        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
+        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
+    ],
+    "content_type": "multipart/mixed",
+    "date": "2016-06-19T14:20:40.000Z",
+    "from_ref": "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed",
+    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
+    "cc_refs": ["email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194"],
+    "subject": "Check out this picture of a cat!",
+    "additional_header_fields": {
+        "Content-Disposition": "inline",
+        "X-Mailer": "Mutt/1.5.23",
+        "X-Originating-IP": "198.51.100.3",
+    },
+    "body_multipart": [
+        {
+            "content_type": "text/plain; charset=utf-8",
+            "content_disposition": "inline",
+            "body": "Cats are funny!",
+        },
+        {
+            "content_type": "image/png",
+            "content_disposition": "attachment; filename=\"tabby.png\"",
+            "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5",
+        },
+        {
+            "content_type": "application/zip",
+            "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
+            "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5",
+        },
+    ],
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_email_msg():
+    store.sink.generate_stix_schema()
+    email_msg_stix_object = stix2.parse(email_msg_dict)
+    store.add(email_msg_stix_object)
+    read_obj = json.loads(store.get(email_msg_stix_object['id']).serialize())
+
+    for attrib in email_msg_dict.keys():
+        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
+                or attrib == "additional_header_fields":  # join multiple tables not implemented yet
+            continue
+        if attrib == "date":
+            assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
+            continue
+        assert email_msg_dict[attrib] == read_obj[attrib]
+
+
+def test_multipart_email_msg():
+    store.sink.generate_stix_schema()
+    multipart_email_msg_stix_object = stix2.parse(multipart_email_msg_dict)
+    store.add(multipart_email_msg_stix_object)
+    read_obj = json.loads(store.get(multipart_email_msg_stix_object['id']).serialize())
+
+    for attrib in multipart_email_msg_dict.keys():
+        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
+                or attrib == "additional_header_fields" or attrib == "body_multipart":  # join multiple tables not implemented yet
+            continue
+        if attrib == "date":
+            assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
+            continue
+        assert multipart_email_msg_dict[attrib] == read_obj[attrib]
+
+# File
+# errors when adding magic_number_hex to store, so ignoring for now
+file_dict = {
+    "type": "file",
+    "spec_version": "2.1",
+    "id": "file--66156fad-2a7d-5237-bbb4-ba1912887cfe",
+    "hashes": {
+        "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a",
+    },
+    "parent_directory_ref": "directory--93c0a9b0-520d-545d-9094-1a08ddf46b05",
+    "name": "qwerty.dll",
+    "size": 25536,
+    "name_enc": "windows-1252",
+    "mime_type": "application/msword",
+    "ctime": "2018-11-23T08:17:27.000Z",
+    "mtime": "2018-11-23T08:17:27.000Z",
+    "atime": "2018-11-23T08:17:27.000Z",
+    "contains_refs": [
+        "file--77156fad-2a0d-5237-bba4-ba1912887cfe",
+    ],
+    "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_file():
+    store.sink.generate_stix_schema()
+    file_stix_object = stix2.parse(file_dict)
+    store.add(file_stix_object)
+    read_obj = store.get(file_stix_object['id'])
+    read_obj = json.loads(store.get(file_stix_object['id']).serialize())
+
+    for attrib in file_dict.keys():
+        if attrib == "contains_refs" or attrib == "hashes":  # join multiple tables not implemented yet
+            continue
+        if attrib == "ctime" or attrib == "mtime" or attrib == "atime":
+            assert stix2.utils.parse_into_datetime(file_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert file_dict[attrib] == read_obj[attrib]
+
+# ipv4 ipv6
+ipv4_dict = {
+    "type": "ipv4-addr",
+    "spec_version": "2.1",
+    "id": "ipv4-addr--ff26c255-6336-5bc5-b98d-13d6226742dd",
+    "value": "198.51.100.3",
+}
+
+ipv6_dict = {
+    "type": "ipv6-addr",
+    "spec_version": "2.1",
+    "id": "ipv6-addr--1e61d36c-a26c-53b7-a80f-2a00161c96b1",
+    "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_ipv4():
+    store.sink.generate_stix_schema()
+    ipv4_stix_object = stix2.parse(ipv4_dict)
+    store.add(ipv4_stix_object)
+    read_obj = store.get(ipv4_stix_object['id'])
+
+    for attrib in ipv4_dict.keys():
+        assert ipv4_dict[attrib] == read_obj[attrib]
+
+
+def test_ipv6():
+    store.sink.generate_stix_schema()
+    ipv6_stix_object = stix2.parse(ipv6_dict)
+    store.add(ipv6_stix_object)
+    read_obj = store.get(ipv6_stix_object['id'])
+
+    for attrib in ipv6_dict.keys():
+        assert ipv6_dict[attrib] == read_obj[attrib]
+
+# Mutex
+mutex_dict = {
+    "type": "mutex",
+    "spec_version": "2.1",
+    "id": "mutex--fba44954-d4e4-5d3b-814c-2b17dd8de300",
+    "name": "__CLEANSWEEP__",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_mutex():
+    store.sink.generate_stix_schema()
+    mutex_stix_object = stix2.parse(mutex_dict)
+    store.add(mutex_stix_object)
+    read_obj = store.get(mutex_stix_object['id'])
+
+    for attrib in mutex_dict.keys():
+        assert mutex_dict[attrib] == read_obj[attrib]
+
+# Network Traffic
+# ipfix property results in a unconsumed value error with the store add
+network_traffic_dict = {
+    "type": "network-traffic",
+    "spec_version": "2.1",
+    "id": "network-traffic--631d7bb1-6bbc-53a6-a6d4-f3c2d35c2734",
+    "src_ref": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53",
+    "dst_ref": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
+    "start": "2018-11-23T08:17:27.000Z",
+    "end": "2018-11-23T08:18:27.000Z",
+    "is_active": False,
+    "src_port": 1000,
+    "dst_port": 1000,
+    "protocols": [
+        "ipv4",
+        "tcp",
+    ],
+    "src_byte_count": 147600,
+    "dst_byte_count": 147600,
+    "src_packets": 100,
+    "dst_packets": 100,
+    "src_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
+    "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03",
+    "encapsulates_refs": [
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4",
+    ],
+    "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_network_traffic():
+    store.sink.generate_stix_schema()
+    network_traffic_stix_object = stix2.parse(network_traffic_dict)
+    store.add(network_traffic_stix_object)
+    read_obj = store.get(network_traffic_stix_object['id'])
+
+    for attrib in network_traffic_dict.keys():
+        if attrib == "encapsulates_refs":  # multiple table join not implemented
+            continue
+        if attrib == "start" or attrib == "end":
+            assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert network_traffic_dict[attrib] == read_obj[attrib]
+
+# Process
+process_dict = {
+    "type": "process",
+    "spec_version": "2.1",
+    "id": "process--f52a906a-0dfc-40bd-92f1-e7778ead38a9",
+    "is_hidden": False,
+    "pid": 1221,
+    "created_time": "2016-01-20T14:11:25.55Z",
+    "cwd": "/tmp/",
+    "environment_variables": {
+        "ENVTEST": "/path/to/bin",
+    },
+    "command_line": "./gedit-bin --new-window",
+    "opened_connection_refs": [
+        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
+    ],
+    "creator_user_ref": "user-account--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
+    "image_ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20",
+    "parent_ref": "process--f52a906a-1dfc-40bd-92f1-e7778ead38a9",
+    "child_refs": [
+        "process--ff2a906a-1dfc-40bd-92f1-e7778ead38a9",
+        "process--fe2a906a-1dfc-40bd-92f1-e7778ead38a9",
+    ],
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_process():
+    store.sink.generate_stix_schema()
+    process_stix_object = stix2.parse(process_dict)
+    store.add(process_stix_object)
+    read_obj = store.get(process_stix_object['id'])
+    read_obj = json.loads(store.get(process_stix_object['id']).serialize())
+
+    for attrib in process_dict.keys():
+        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables":
+            # join multiple tables not implemented yet
+            continue
+        if attrib == "created_time":
+            assert stix2.utils.parse_into_datetime(process_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert process_dict[attrib] == read_obj[attrib]
+
+# Software
+software_dict = {
+    "type": "software",
+    "spec_version": "2.1",
+    "id": "software--a1827f6d-ca53-5605-9e93-4316cd22a00a",
+    "name": "Word",
+    "cpe": "cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
+    "version": "2002",
+    "vendor": "Microsoft",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_software():
+    store.sink.generate_stix_schema()
+    software_stix_object = stix2.parse(software_dict)
+    store.add(software_stix_object)
+    read_obj = store.get(software_stix_object['id'])
+    read_obj = json.loads(store.get(software_stix_object['id']).serialize())
+
+    for attrib in software_dict.keys():
+        assert software_dict[attrib] == read_obj[attrib]
+
+
+# URL
+url_dict = {
+    "type": "url",
+    "id": "url--a5477287-23ac-5971-a010-5c287877fa60",
+    "value": "https://example.com/research/index.html",
+}
+
+store = RelationalDBStore(
+        "postgresql://postgres:admin@localhost/postgres",
+        False,
+        None,
+        True,
+)
+
+
+def test_url():
+    store.sink.generate_stix_schema()
+    url_stix_object = stix2.parse(url_dict)
+    store.add(url_stix_object)
+    read_obj = json.loads(store.get(url_stix_object['id']).serialize())
+
+    for attrib in url_dict.keys():
+        assert url_dict[attrib] == read_obj[attrib]
+
+# User Account
+user_account_dict = {
+    "type": "user-account",
+    "spec_version": "2.1",
+    "id": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
+    "user_id": "1001",
+    "credential": "password",
+    "account_login": "jdoe",
+    "account_type": "unix",
+    "display_name": "John Doe",
+    "is_service_account": False,
+    "is_privileged": False,
+    "can_escalate_privs": True,
+    "is_disabled": False,
+    "account_created": "2016-01-20T12:31:12Z",
+    "account_expires": "2018-01-20T12:31:12Z",
+    "credential_last_changed": "2016-01-20T14:27:43Z",
+    "account_first_login": "2016-01-20T14:26:07Z",
+    "account_last_login": "2016-07-22T16:08:28Z",
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_user_account():
+    store.sink.generate_stix_schema()
+    user_account_stix_object = stix2.parse(user_account_dict)
+    store.add(user_account_stix_object)
+    read_obj = json.loads(store.get(user_account_stix_object['id']).serialize())
+
+    for attrib in user_account_dict.keys():
+        if attrib == "account_created" or attrib == "account_expires" \
+                or attrib == "credential_last_changed" or attrib == "account_first_login" \
+                or attrib == "account_last_login":
+            assert stix2.utils.parse_into_datetime(user_account_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
+            continue
+        assert user_account_dict[attrib] == read_obj[attrib]
+
+# Windows Registry
+windows_registry_dict = {
+    "type": "windows-registry-key",
+    "spec_version": "2.1",
+    "id": "windows-registry-key--2ba37ae7-2745-5082-9dfd-9486dad41016",
+    "key": "hkey_local_machine\\system\\bar\\foo",
+    "values": [
+        {
+            "name": "Foo",
+            "data": "qwerty",
+            "data_type": "REG_SZ",
+        },
+        {
+            "name": "Bar",
+            "data": "42",
+            "data_type": "REG_DWORD",
+        },
+    ],
+    "modified_time": "2018-01-20T12:31:12Z",
+    "creator_user_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
+    "number_of_subkeys": 2,
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_windows_registry():
+    store.sink.generate_stix_schema()
+    windows_registry_stix_object = stix2.parse(windows_registry_dict)
+    store.add(windows_registry_stix_object)
+    read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
+
+    for attrib in windows_registry_dict.keys():
+        if attrib == "values":  # skip multiple table join
+            continue
+        if attrib == "modified_time":
+            assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(
+                read_obj[attrib],
+            )
+            continue
+        assert windows_registry_dict[attrib] == read_obj[attrib]
+
+# x509 Certificate
+basic_x509_certificate_dict = {
+    "type": "x509-certificate",
+    "spec_version": "2.1",
+    "id": "x509-certificate--463d7b2a-8516-5a50-a3d7-6f801465d5de",
+    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification  \
+    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
+    "validity_not_before": "2016-03-12T12:00:00Z",
+    "validity_not_after": "2016-08-21T12:00:00Z",
+    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
+    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
+    "serial_number": "36:f7:d4:32:f4:ab:70:ea:d3:ce:98:6e:ea:99:93:49:32:0a:b7:06",
+}
+
+extensions_x509_certificate_dict = {
+    "type": "x509-certificate",
+    "spec_version": "2.1",
+    "id": "x509-certificate--b595eaf0-0b28-5dad-9e8e-0fab9c1facc9",
+    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification \
+    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
+    "validity_not_before": "2016-03-12T12:00:00Z",
+    "validity_not_after": "2016-08-21T12:00:00Z",
+    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
+    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
+    "serial_number": "02:08:87:83:f2:13:58:1f:79:52:1e:66:90:0a:02:24:c9:6b:c7:dc",
+    "x509_v3_extensions": {
+        "basic_constraints": "critical,CA:TRUE, pathlen:0",
+        "name_constraints": "permitted;IP:192.168.0.0/255.255.0.0",
+        "policy_constraints": "requireExplicitPolicy:3",
+        "key_usage": "critical, keyCertSign",
+        "extended_key_usage": "critical,codeSigning,1.2.3.4",
+        "subject_key_identifier": "hash",
+        "authority_key_identifier": "keyid,issuer",
+        "subject_alternative_name": "email:my@other.address,RID:1.2.3.4",
+        "issuer_alternative_name": "issuer:copy",
+        "crl_distribution_points": "URI:http://myhost.com/myca.crl",
+        "inhibit_any_policy": "2",
+        "private_key_usage_period_not_before": "2016-03-12T12:00:00Z",
+        "private_key_usage_period_not_after": "2018-03-12T12:00:00Z",
+        "certificate_policies": "1.2.4.5, 1.1.3.4",
+    },
+}
+
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True,
+    force_recreate=True
+)
+
+
+def test_basic_x509_certificate():
+    store.sink.generate_stix_schema()
+    basic_x509_certificate_stix_object = stix2.parse(basic_x509_certificate_dict)
+    store.add(basic_x509_certificate_stix_object)
+    read_obj = json.loads(store.get(basic_x509_certificate_stix_object['id']).serialize())
+
+    for attrib in basic_x509_certificate_dict.keys():
+        if attrib == "validity_not_before" or attrib == "validity_not_after":
+            assert stix2.utils.parse_into_datetime(
+                basic_x509_certificate_dict[attrib],
+            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert basic_x509_certificate_dict[attrib] == read_obj[attrib]
+
+
+def test_x509_certificate_with_extensions():
+    store.sink.generate_stix_schema()
+    extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
+    store.add(extensions_x509_certificate_stix_object)
+    read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())
+
+    for attrib in extensions_x509_certificate_dict.keys():
+        if attrib == "x509_v3_extensions":  # skipping multi-table join
+            continue
+        if attrib == "validity_not_before" or attrib == "validity_not_after":
+            assert stix2.utils.parse_into_datetime(
+                extensions_x509_certificate_dict[attrib],
+            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
+            continue
+        assert extensions_x509_certificate_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_directory.py b/stix2/test/v21/test_directory.py
deleted file mode 100644
index 2f413866..00000000
--- a/stix2/test/v21/test_directory.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-
-directory_dict = {
-    "type": "directory",
-    "spec_version": "2.1",
-    "id": "directory--17c909b1-521d-545d-9094-1a08ddf46b05",
-    "ctime": "2018-11-23T08:17:27.000Z",
-    "mtime": "2018-11-23T08:17:27.000Z",
-    "path": "C:\\Windows\\System32",
-    "path_enc": "cGF0aF9lbmM",
-    "contains_refs": [
-        "directory--94c0a9b0-520d-545d-9094-1a08ddf46b05",
-        "file--95c0a9b0-520d-545d-9094-1a08ddf46b05",
-    ],
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_directory():
-    store.sink.generate_stix_schema()
-    directory_obj = stix2.parse(directory_dict)
-    store.add(directory_obj)
-    read_obj = json.loads(store.get(directory_obj['id']).serialize())
-
-    for attrib in directory_dict.keys():
-        if attrib == "contains_refs":  # TODO remove skip once we can pull from table join
-            continue
-        if attrib == "ctime" or attrib == "mtime":  # convert both into stix2 date format for consistency
-            assert stix2.utils.parse_into_datetime(directory_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert directory_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_domain_name.py b/stix2/test/v21/test_domain_name.py
deleted file mode 100644
index 93f30d54..00000000
--- a/stix2/test/v21/test_domain_name.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-
-domain_name_dict = {
-    "type": "domain-name",
-    "spec_version": "2.1",
-    "id": "domain-name--3c10e93f-798e-5a26-a0c1-08156efab7f5",
-    "value": "example.com",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_autonomous_system():
-    store.sink.generate_stix_schema()
-    domain_name_obj = stix2.parse(domain_name_dict)
-    store.add(domain_name_obj)
-    read_obj = json.loads(store.get(domain_name_obj['id']).serialize())
-
-    for attrib in domain_name_dict.keys():
-        assert domain_name_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_email_address.py b/stix2/test/v21/test_email_address.py
deleted file mode 100644
index 6a00daef..00000000
--- a/stix2/test/v21/test_email_address.py
+++ /dev/null
@@ -1,31 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-email_addr_dict = {
-    "type": "email-addr",
-    "spec_version": "2.1",
-    "id": "email-addr--2d77a846-6264-5d51-b586-e43822ea1ea3",
-    "value": "john@example.com",
-    "display_name": "John Doe",
-    "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_email_addr():
-    store.sink.generate_stix_schema()
-    email_addr_stix_object = stix2.parse(email_addr_dict)
-    store.add(email_addr_stix_object)
-    read_obj = json.loads(store.get(email_addr_stix_object['id']).serialize())
-
-    for attrib in email_addr_dict.keys():
-        assert email_addr_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_email_message.py b/stix2/test/v21/test_email_message.py
deleted file mode 100644
index 038b3274..00000000
--- a/stix2/test/v21/test_email_message.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-email_msg_dict = {
-    "type": "email-message",
-    "spec_version": "2.1",
-    "id": "email-message--8c57a381-2a17-5e61-8754-5ef96efb286c",
-    "from_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5dda",
-    "sender_ref": "email-addr--9b7e29b3-fd8d-562e-b3f0-8fc8134f5eeb",
-    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
-    "cc_refs": [
-        "email-addr--d2b3bf0c-f02a-51a1-8102-11aba7959868",
-        "email-addr--d3b3bf0c-f02a-51a1-8102-11aba7959868",
-    ],
-    "bcc_refs": [
-        "email-addr--d4b3bf0c-f02a-51a1-8102-11aba7959868",
-        "email-addr--d5b3bf0c-f02a-51a1-8102-11aba7959868",
-    ],
-    "message_id": "message01",
-    "is_multipart": False,
-    "date": "2004-04-19T12:22:23.000Z",
-    "subject": "Did you see this?",
-    "received_lines": [
-        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
-        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
-        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
-    ],
-    "additional_header_fields": {
-        "Reply-To": [
-            "steve@example.com",
-            "jane@example.com",
-        ],
-    },
-    "body": "message body",
-    "raw_email_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
-}
-
-multipart_email_msg_dict = {
-    "type": "email-message",
-    "spec_version": "2.1",
-    "id": "email-message--ef9b4b7f-14c8-5955-8065-020e0316b559",
-    "is_multipart": True,
-    "received_lines": [
-        "from mail.example.com ([198.51.100.3]) by smtp.gmail.com with ESMTPSA id \
-        q23sm23309939wme.17.2016.07.19.07.20.32 (version=TLS1_2 cipher=ECDHE-RSA-AES128-GCM-SHA256 \
-        bits=128/128); Tue, 19 Jul 2016 07:20:40 -0700 (PDT)",
-    ],
-    "content_type": "multipart/mixed",
-    "date": "2016-06-19T14:20:40.000Z",
-    "from_ref": "email-addr--89f52ea8-d6ef-51e9-8fce-6a29236436ed",
-    "to_refs": ["email-addr--d1b3bf0c-f02a-51a1-8102-11aba7959868"],
-    "cc_refs": ["email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194"],
-    "subject": "Check out this picture of a cat!",
-    "additional_header_fields": {
-        "Content-Disposition": "inline",
-        "X-Mailer": "Mutt/1.5.23",
-        "X-Originating-IP": "198.51.100.3",
-    },
-    "body_multipart": [
-        {
-            "content_type": "text/plain; charset=utf-8",
-            "content_disposition": "inline",
-            "body": "Cats are funny!",
-        },
-        {
-            "content_type": "image/png",
-            "content_disposition": "attachment; filename=\"tabby.png\"",
-            "body_raw_ref": "artifact--4cce66f8-6eaa-53cb-85d5-3a85fca3a6c5",
-        },
-        {
-            "content_type": "application/zip",
-            "content_disposition": "attachment; filename=\"tabby_pics.zip\"",
-            "body_raw_ref": "file--6ce09d9c-0ad3-5ebf-900c-e3cb288955b5",
-        },
-    ],
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_email_msg():
-    store.sink.generate_stix_schema()
-    email_msg_stix_object = stix2.parse(email_msg_dict)
-    store.add(email_msg_stix_object)
-    read_obj = json.loads(store.get(email_msg_stix_object['id']).serialize())
-
-    for attrib in email_msg_dict.keys():
-        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-                or attrib == "additional_header_fields":  # join multiple tables not implemented yet
-            continue
-        if attrib == "date":
-            assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
-                read_obj[attrib],
-            )
-            continue
-        assert email_msg_dict[attrib] == read_obj[attrib]
-
-
-def test_multipart_email_msg():
-    store.sink.generate_stix_schema()
-    multipart_email_msg_stix_object = stix2.parse(multipart_email_msg_dict)
-    store.add(multipart_email_msg_stix_object)
-    read_obj = json.loads(store.get(multipart_email_msg_stix_object['id']).serialize())
-
-    for attrib in multipart_email_msg_dict.keys():
-        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-                or attrib == "additional_header_fields" or attrib == "body_multipart":  # join multiple tables not implemented yet
-            continue
-        if attrib == "date":
-            assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
-                read_obj[attrib],
-            )
-            continue
-        assert multipart_email_msg_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_file.py b/stix2/test/v21/test_file.py
deleted file mode 100644
index 647f81e5..00000000
--- a/stix2/test/v21/test_file.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-# errors when adding magic_number_hex to store, so ignoring for now
-
-file_dict = {
-    "type": "file",
-    "spec_version": "2.1",
-    "id": "file--66156fad-2a7d-5237-bbb4-ba1912887cfe",
-    "hashes": {
-        "SHA-256": "ceafbfd424be2ca4a5f0402cae090dda2fb0526cf521b60b60077c0f622b285a",
-    },
-    "parent_directory_ref": "directory--93c0a9b0-520d-545d-9094-1a08ddf46b05",
-    "name": "qwerty.dll",
-    "size": 25536,
-    "name_enc": "windows-1252",
-    "mime_type": "application/msword",
-    "ctime": "2018-11-23T08:17:27.000Z",
-    "mtime": "2018-11-23T08:17:27.000Z",
-    "atime": "2018-11-23T08:17:27.000Z",
-    "contains_refs": [
-        "file--77156fad-2a0d-5237-bba4-ba1912887cfe",
-    ],
-    "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_file():
-    store.sink.generate_stix_schema()
-    file_stix_object = stix2.parse(file_dict)
-    store.add(file_stix_object)
-    read_obj = store.get(file_stix_object['id'])
-    read_obj = json.loads(store.get(file_stix_object['id']).serialize())
-
-    for attrib in file_dict.keys():
-        if attrib == "contains_refs" or attrib == "hashes":  # join multiple tables not implemented yet
-            continue
-        if attrib == "ctime" or attrib == "mtime" or attrib == "atime":
-            assert stix2.utils.parse_into_datetime(file_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert file_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_ipv4_ipv6.py b/stix2/test/v21/test_ipv4_ipv6.py
deleted file mode 100644
index c32197d4..00000000
--- a/stix2/test/v21/test_ipv4_ipv6.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-ipv4_dict = {
-    "type": "ipv4-addr",
-    "spec_version": "2.1",
-    "id": "ipv4-addr--ff26c255-6336-5bc5-b98d-13d6226742dd",
-    "value": "198.51.100.3",
-}
-
-ipv6_dict = {
-    "type": "ipv6-addr",
-    "spec_version": "2.1",
-    "id": "ipv6-addr--1e61d36c-a26c-53b7-a80f-2a00161c96b1",
-    "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_ipv4():
-    store.sink.generate_stix_schema()
-    ipv4_stix_object = stix2.parse(ipv4_dict)
-    store.add(ipv4_stix_object)
-    read_obj = store.get(ipv4_stix_object['id'])
-
-    for attrib in ipv4_dict.keys():
-        assert ipv4_dict[attrib] == read_obj[attrib]
-
-
-def test_ipv6():
-    store.sink.generate_stix_schema()
-    ipv6_stix_object = stix2.parse(ipv6_dict)
-    store.add(ipv6_stix_object)
-    read_obj = store.get(ipv6_stix_object['id'])
-
-    for attrib in ipv6_dict.keys():
-        assert ipv6_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_mutex.py b/stix2/test/v21/test_mutex.py
deleted file mode 100644
index 55fdd5d2..00000000
--- a/stix2/test/v21/test_mutex.py
+++ /dev/null
@@ -1,27 +0,0 @@
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-mutex_dict = {
-    "type": "mutex",
-    "spec_version": "2.1",
-    "id": "mutex--fba44954-d4e4-5d3b-814c-2b17dd8de300",
-    "name": "__CLEANSWEEP__",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_mutex():
-    store.sink.generate_stix_schema()
-    mutex_stix_object = stix2.parse(mutex_dict)
-    store.add(mutex_stix_object)
-    read_obj = store.get(mutex_stix_object['id'])
-
-    for attrib in mutex_dict.keys():
-        assert mutex_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_network_traffic.py b/stix2/test/v21/test_network_traffic.py
deleted file mode 100644
index ddb47a49..00000000
--- a/stix2/test/v21/test_network_traffic.py
+++ /dev/null
@@ -1,55 +0,0 @@
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-# ipfix property results in a unconsumed value error with the store add
-
-network_traffic_dict = {
-    "type": "network-traffic",
-    "spec_version": "2.1",
-    "id": "network-traffic--631d7bb1-6bbc-53a6-a6d4-f3c2d35c2734",
-    "src_ref": "ipv4-addr--4d22aae0-2bf9-5427-8819-e4f6abf20a53",
-    "dst_ref": "ipv4-addr--03b708d9-7761-5523-ab75-5ea096294a68",
-    "start": "2018-11-23T08:17:27.000Z",
-    "end": "2018-11-23T08:18:27.000Z",
-    "is_active": False,
-    "src_port": 1000,
-    "dst_port": 1000,
-    "protocols": [
-        "ipv4",
-        "tcp",
-    ],
-    "src_byte_count": 147600,
-    "dst_byte_count": 147600,
-    "src_packets": 100,
-    "dst_packets": 100,
-    "src_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b02",
-    "dst_payload_ref": "artifact--3857f78d-7d16-5092-99fe-ecff58408b03",
-    "encapsulates_refs": [
-        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
-        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a4",
-    ],
-    "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_network_traffic():
-    store.sink.generate_stix_schema()
-    network_traffic_stix_object = stix2.parse(network_traffic_dict)
-    store.add(network_traffic_stix_object)
-    read_obj = store.get(network_traffic_stix_object['id'])
-
-    for attrib in network_traffic_dict.keys():
-        if attrib == "encapsulates_refs":  # multiple table join not implemented
-            continue
-        if attrib == "start" or attrib == "end":
-            assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert network_traffic_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_process.py b/stix2/test/v21/test_process.py
deleted file mode 100644
index 519fd2d1..00000000
--- a/stix2/test/v21/test_process.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-process_dict = {
-    "type": "process",
-    "spec_version": "2.1",
-    "id": "process--f52a906a-0dfc-40bd-92f1-e7778ead38a9",
-    "is_hidden": False,
-    "pid": 1221,
-    "created_time": "2016-01-20T14:11:25.55Z",
-    "cwd": "/tmp/",
-    "environment_variables": {
-        "ENVTEST": "/path/to/bin",
-    },
-    "command_line": "./gedit-bin --new-window",
-    "opened_connection_refs": [
-        "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a3",
-    ],
-    "creator_user_ref": "user-account--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
-    "image_ref": "file--e04f22d1-be2c-59de-add8-10f61d15fe20",
-    "parent_ref": "process--f52a906a-1dfc-40bd-92f1-e7778ead38a9",
-    "child_refs": [
-        "process--ff2a906a-1dfc-40bd-92f1-e7778ead38a9",
-        "process--fe2a906a-1dfc-40bd-92f1-e7778ead38a9",
-    ],
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_process():
-    store.sink.generate_stix_schema()
-    process_stix_object = stix2.parse(process_dict)
-    store.add(process_stix_object)
-    read_obj = store.get(process_stix_object['id'])
-    read_obj = json.loads(store.get(process_stix_object['id']).serialize())
-
-    for attrib in process_dict.keys():
-        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables":
-            # join multiple tables not implemented yet
-            continue
-        if attrib == "created_time":
-            assert stix2.utils.parse_into_datetime(process_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert process_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_software.py b/stix2/test/v21/test_software.py
deleted file mode 100644
index 896e9c2a..00000000
--- a/stix2/test/v21/test_software.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-software_dict = {
-    "type": "software",
-    "spec_version": "2.1",
-    "id": "software--a1827f6d-ca53-5605-9e93-4316cd22a00a",
-    "name": "Word",
-    "cpe": "cpe:2.3:a:microsoft:word:2000:*:*:*:*:*:*:*",
-    "version": "2002",
-    "vendor": "Microsoft",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_software():
-    store.sink.generate_stix_schema()
-    software_stix_object = stix2.parse(software_dict)
-    store.add(software_stix_object)
-    read_obj = store.get(software_stix_object['id'])
-    read_obj = json.loads(store.get(software_stix_object['id']).serialize())
-
-    for attrib in software_dict.keys():
-        assert software_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_url.py b/stix2/test/v21/test_url.py
deleted file mode 100644
index 838cbfbb..00000000
--- a/stix2/test/v21/test_url.py
+++ /dev/null
@@ -1,28 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-url_dict = {
-    "type": "url",
-    "id": "url--a5477287-23ac-5971-a010-5c287877fa60",
-    "value": "https://example.com/research/index.html",
-}
-
-store = RelationalDBStore(
-        "postgresql://postgres:admin@localhost/postgres",
-        False,
-        None,
-        True,
-)
-
-
-def test_url():
-    store.sink.generate_stix_schema()
-    url_stix_object = stix2.parse(url_dict)
-    store.add(url_stix_object)
-    read_obj = json.loads(store.get(url_stix_object['id']).serialize())
-
-    for attrib in url_dict.keys():
-        assert url_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_user_account.py b/stix2/test/v21/test_user_account.py
deleted file mode 100644
index 374c2377..00000000
--- a/stix2/test/v21/test_user_account.py
+++ /dev/null
@@ -1,49 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-user_account_dict = {
-    "type": "user-account",
-    "spec_version": "2.1",
-    "id": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
-    "user_id": "1001",
-    "credential": "password",
-    "account_login": "jdoe",
-    "account_type": "unix",
-    "display_name": "John Doe",
-    "is_service_account": False,
-    "is_privileged": False,
-    "can_escalate_privs": True,
-    "is_disabled": False,
-    "account_created": "2016-01-20T12:31:12Z",
-    "account_expires": "2018-01-20T12:31:12Z",
-    "credential_last_changed": "2016-01-20T14:27:43Z",
-    "account_first_login": "2016-01-20T14:26:07Z",
-    "account_last_login": "2016-07-22T16:08:28Z",
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_user_account():
-    store.sink.generate_stix_schema()
-    user_account_stix_object = stix2.parse(user_account_dict)
-    store.add(user_account_stix_object)
-    read_obj = json.loads(store.get(user_account_stix_object['id']).serialize())
-
-    for attrib in user_account_dict.keys():
-        if attrib == "account_created" or attrib == "account_expires" \
-                or attrib == "credential_last_changed" or attrib == "account_first_login" \
-                or attrib == "account_last_login":
-            assert stix2.utils.parse_into_datetime(user_account_dict[attrib]) == stix2.utils.parse_into_datetime(
-                read_obj[attrib],
-            )
-            continue
-        assert user_account_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_windows_registry.py b/stix2/test/v21/test_windows_registry.py
deleted file mode 100644
index f2864548..00000000
--- a/stix2/test/v21/test_windows_registry.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-windows_registry_dict = {
-    "type": "windows-registry-key",
-    "spec_version": "2.1",
-    "id": "windows-registry-key--2ba37ae7-2745-5082-9dfd-9486dad41016",
-    "key": "hkey_local_machine\\system\\bar\\foo",
-    "values": [
-        {
-            "name": "Foo",
-            "data": "qwerty",
-            "data_type": "REG_SZ",
-        },
-        {
-            "name": "Bar",
-            "data": "42",
-            "data_type": "REG_DWORD",
-        },
-    ],
-    "modified_time": "2018-01-20T12:31:12Z",
-    "creator_user_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
-    "number_of_subkeys": 2,
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_windows_registry():
-    store.sink.generate_stix_schema()
-    windows_registry_stix_object = stix2.parse(windows_registry_dict)
-    store.add(windows_registry_stix_object)
-    read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
-
-    for attrib in windows_registry_dict.keys():
-        if attrib == "values":  # skip multiple table join
-            continue
-        if attrib == "modified_time":
-            assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(
-                read_obj[attrib],
-            )
-            continue
-        assert windows_registry_dict[attrib] == read_obj[attrib]
diff --git a/stix2/test/v21/test_x509_certificates.py b/stix2/test/v21/test_x509_certificates.py
deleted file mode 100644
index 1847dbc2..00000000
--- a/stix2/test/v21/test_x509_certificates.py
+++ /dev/null
@@ -1,86 +0,0 @@
-import json
-
-import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
-import stix2.properties
-
-basic_x509_certificate_dict = {
-    "type": "x509-certificate",
-    "spec_version": "2.1",
-    "id": "x509-certificate--463d7b2a-8516-5a50-a3d7-6f801465d5de",
-    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification  \
-    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
-    "validity_not_before": "2016-03-12T12:00:00Z",
-    "validity_not_after": "2016-08-21T12:00:00Z",
-    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
-    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
-    "serial_number": "36:f7:d4:32:f4:ab:70:ea:d3:ce:98:6e:ea:99:93:49:32:0a:b7:06",
-}
-
-extensions_x509_certificate_dict = {
-    "type": "x509-certificate",
-    "spec_version": "2.1",
-    "id": "x509-certificate--b595eaf0-0b28-5dad-9e8e-0fab9c1facc9",
-    "issuer": "C=ZA, ST=Western Cape, L=Cape Town, O=Thawte Consulting cc, OU=Certification \
-    Services Division, CN=Thawte Server CA/emailAddress=server-certs@thawte.com",
-    "validity_not_before": "2016-03-12T12:00:00Z",
-    "validity_not_after": "2016-08-21T12:00:00Z",
-    "subject": "C=US, ST=Maryland, L=Pasadena, O=Brent Baccala, OU=FreeSoft, \
-    CN=www.freesoft.org/emailAddress=baccala@freesoft.org",
-    "serial_number": "02:08:87:83:f2:13:58:1f:79:52:1e:66:90:0a:02:24:c9:6b:c7:dc",
-    "x509_v3_extensions": {
-        "basic_constraints": "critical,CA:TRUE, pathlen:0",
-        "name_constraints": "permitted;IP:192.168.0.0/255.255.0.0",
-        "policy_constraints": "requireExplicitPolicy:3",
-        "key_usage": "critical, keyCertSign",
-        "extended_key_usage": "critical,codeSigning,1.2.3.4",
-        "subject_key_identifier": "hash",
-        "authority_key_identifier": "keyid,issuer",
-        "subject_alternative_name": "email:my@other.address,RID:1.2.3.4",
-        "issuer_alternative_name": "issuer:copy",
-        "crl_distribution_points": "URI:http://myhost.com/myca.crl",
-        "inhibit_any_policy": "2",
-        "private_key_usage_period_not_before": "2016-03-12T12:00:00Z",
-        "private_key_usage_period_not_after": "2018-03-12T12:00:00Z",
-        "certificate_policies": "1.2.4.5, 1.1.3.4",
-    },
-}
-
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-)
-
-
-def test_basic_x509_certificate():
-    store.sink.generate_stix_schema()
-    basic_x509_certificate_stix_object = stix2.parse(basic_x509_certificate_dict)
-    store.add(basic_x509_certificate_stix_object)
-    read_obj = json.loads(store.get(basic_x509_certificate_stix_object['id']).serialize())
-
-    for attrib in basic_x509_certificate_dict.keys():
-        if attrib == "validity_not_before" or attrib == "validity_not_after":
-            assert stix2.utils.parse_into_datetime(
-                basic_x509_certificate_dict[attrib],
-            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert basic_x509_certificate_dict[attrib] == read_obj[attrib]
-
-
-def test_x509_certificate_with_extensions():
-    store.sink.generate_stix_schema()
-    extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
-    store.add(extensions_x509_certificate_stix_object)
-    read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())
-
-    for attrib in extensions_x509_certificate_dict.keys():
-        if attrib == "x509_v3_extensions":  # skipping multi-table join
-            continue
-        if attrib == "validity_not_before" or attrib == "validity_not_after":
-            assert stix2.utils.parse_into_datetime(
-                extensions_x509_certificate_dict[attrib],
-            ) == stix2.utils.parse_into_datetime(read_obj[attrib])
-            continue
-        assert extensions_x509_certificate_dict[attrib] == read_obj[attrib]

From 8285a2fc6e111b083268ac48f6bd3b3292225dcf Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Tue, 16 Apr 2024 12:06:42 -0400
Subject: [PATCH 084/132] removing multiple store var

---
 .../test/v21/test_datastore_relational_db.py  | 127 +-----------------
 1 file changed, 7 insertions(+), 120 deletions(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 746b3333..39524ac0 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -4,6 +4,13 @@
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
+store = RelationalDBStore(
+    "postgresql://postgres:admin@localhost/postgres",
+    False,
+    None,
+    True
+)
+
 # Artifacts
 basic_artifact_dict = {
     "type": "artifact",
@@ -27,14 +34,6 @@
     "decryption_key": "My voice is my passport",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_basic_artifact():
     store.sink.generate_stix_schema()
@@ -67,14 +66,6 @@ def test_encrypted_artifact():
     "rir": "ARIN",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_autonomous_system():
     store.sink.generate_stix_schema()
@@ -100,14 +91,6 @@ def test_autonomous_system():
     ],
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_directory():
     store.sink.generate_stix_schema()
@@ -131,14 +114,6 @@ def test_directory():
     "value": "example.com",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_autonomous_system():
     store.sink.generate_stix_schema()
@@ -159,14 +134,6 @@ def test_autonomous_system():
     "belongs_to_ref": "user-account--0d5b424b-93b8-5cd8-ac36-306e1789d63c",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_email_addr():
     store.sink.generate_stix_schema()
@@ -252,14 +219,6 @@ def test_email_addr():
     ],
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_email_msg():
     store.sink.generate_stix_schema()
@@ -319,14 +278,6 @@ def test_multipart_email_msg():
     "content_ref": "artifact--cb37bcf8-9846-5ab4-8662-75c1bf6e63ee",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_file():
     store.sink.generate_stix_schema()
@@ -358,14 +309,6 @@ def test_file():
     "value": "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_ipv4():
     store.sink.generate_stix_schema()
@@ -394,14 +337,6 @@ def test_ipv6():
     "name": "__CLEANSWEEP__",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_mutex():
     store.sink.generate_stix_schema()
@@ -442,14 +377,6 @@ def test_mutex():
     "encapsulated_by_ref": "network-traffic--53e0bf48-2eee-5c03-8bde-ed7049d2c0a5",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_network_traffic():
     store.sink.generate_stix_schema()
@@ -490,14 +417,6 @@ def test_network_traffic():
     ],
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_process():
     store.sink.generate_stix_schema()
@@ -526,14 +445,6 @@ def test_process():
     "vendor": "Microsoft",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_software():
     store.sink.generate_stix_schema()
@@ -591,14 +502,6 @@ def test_url():
     "account_last_login": "2016-07-22T16:08:28Z",
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_user_account():
     store.sink.generate_stix_schema()
@@ -639,14 +542,6 @@ def test_user_account():
     "number_of_subkeys": 2,
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_windows_registry():
     store.sink.generate_stix_schema()
@@ -707,14 +602,6 @@ def test_windows_registry():
     },
 }
 
-store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
-    True,
-    force_recreate=True
-)
-
 
 def test_basic_x509_certificate():
     store.sink.generate_stix_schema()

From b613add07df4f8c31ac18b2902c2fba74fcdfa06 Mon Sep 17 00:00:00 2001
From: ryan <ryanxu@wustl.edu>
Date: Tue, 16 Apr 2024 12:16:05 -0400
Subject: [PATCH 085/132] Fix styling

---
 stix2/properties.py                            |  4 ++--
 stix2/test/v21/test_datastore_relational_db.py | 18 ++++++++++++++++--
 2 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 55358d41..f8e79ac9 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -746,7 +746,7 @@ def __init__(self, allowed, **kwargs):
 
     def clean(self, value, allow_custom, strict=False):
         cleaned_value, _ = super(OpenVocabProperty, self).clean(
-            value, allow_custom, strict
+            value, allow_custom, strict,
         )
 
         # Disabled: it was decided that enforcing this is too strict (might
@@ -755,7 +755,7 @@ def clean(self, value, allow_custom, strict=False):
         #
         if strict is True:
             has_custom = cleaned_value not in self.allowed
-            
+
             if not allow_custom and has_custom:
                 raise CustomContentError(
                     "custom value in open vocab: '{}'".format(cleaned_value),
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 39524ac0..8003843b 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -8,7 +8,7 @@
     "postgresql://postgres:admin@localhost/postgres",
     False,
     None,
-    True
+    True,
 )
 
 # Artifacts
@@ -56,6 +56,7 @@ def test_encrypted_artifact():
             continue
         assert encrypted_artifact_dict[attrib] == read_obj[attrib]
 
+
 # Autonomous System
 as_dict = {
     "type": "autonomous-system",
@@ -76,6 +77,7 @@ def test_autonomous_system():
     for attrib in as_dict.keys():
         assert as_dict[attrib] == read_obj[attrib]
 
+
 # Directory
 directory_dict = {
     "type": "directory",
@@ -106,6 +108,7 @@ def test_directory():
             continue
         assert directory_dict[attrib] == read_obj[attrib]
 
+
 # Domain Name
 domain_name_dict = {
     "type": "domain-name",
@@ -115,7 +118,7 @@ def test_directory():
 }
 
 
-def test_autonomous_system():
+def test_domain_name():
     store.sink.generate_stix_schema()
     domain_name_obj = stix2.parse(domain_name_dict)
     store.add(domain_name_obj)
@@ -124,6 +127,7 @@ def test_autonomous_system():
     for attrib in domain_name_dict.keys():
         assert domain_name_dict[attrib] == read_obj[attrib]
 
+
 # Email Address
 email_addr_dict = {
     "type": "email-addr",
@@ -144,6 +148,7 @@ def test_email_addr():
     for attrib in email_addr_dict.keys():
         assert email_addr_dict[attrib] == read_obj[attrib]
 
+
 # Email Message
 email_msg_dict = {
     "type": "email-message",
@@ -255,6 +260,7 @@ def test_multipart_email_msg():
             continue
         assert multipart_email_msg_dict[attrib] == read_obj[attrib]
 
+
 # File
 # errors when adding magic_number_hex to store, so ignoring for now
 file_dict = {
@@ -294,6 +300,7 @@ def test_file():
             continue
         assert file_dict[attrib] == read_obj[attrib]
 
+
 # ipv4 ipv6
 ipv4_dict = {
     "type": "ipv4-addr",
@@ -329,6 +336,7 @@ def test_ipv6():
     for attrib in ipv6_dict.keys():
         assert ipv6_dict[attrib] == read_obj[attrib]
 
+
 # Mutex
 mutex_dict = {
     "type": "mutex",
@@ -347,6 +355,7 @@ def test_mutex():
     for attrib in mutex_dict.keys():
         assert mutex_dict[attrib] == read_obj[attrib]
 
+
 # Network Traffic
 # ipfix property results in a unconsumed value error with the store add
 network_traffic_dict = {
@@ -392,6 +401,7 @@ def test_network_traffic():
             continue
         assert network_traffic_dict[attrib] == read_obj[attrib]
 
+
 # Process
 process_dict = {
     "type": "process",
@@ -434,6 +444,7 @@ def test_process():
             continue
         assert process_dict[attrib] == read_obj[attrib]
 
+
 # Software
 software_dict = {
     "type": "software",
@@ -481,6 +492,7 @@ def test_url():
     for attrib in url_dict.keys():
         assert url_dict[attrib] == read_obj[attrib]
 
+
 # User Account
 user_account_dict = {
     "type": "user-account",
@@ -519,6 +531,7 @@ def test_user_account():
             continue
         assert user_account_dict[attrib] == read_obj[attrib]
 
+
 # Windows Registry
 windows_registry_dict = {
     "type": "windows-registry-key",
@@ -559,6 +572,7 @@ def test_windows_registry():
             continue
         assert windows_registry_dict[attrib] == read_obj[attrib]
 
+
 # x509 Certificate
 basic_x509_certificate_dict = {
     "type": "x509-certificate",

From 0513ca0ffac48287cc6c83f8806404dc208b5a3f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:33:03 -0400
Subject: [PATCH 086/132] remove extra store

---
 .../test/v21/test_datastore_relational_db.py  | 33 ++-----------------
 1 file changed, 3 insertions(+), 30 deletions(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 8003843b..f9626dcd 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -5,10 +5,10 @@
 import stix2.properties
 
 store = RelationalDBStore(
-    "postgresql://postgres:admin@localhost/postgres",
-    False,
-    None,
+    "postgresql://localhost/stix-data-sink",
     True,
+    None,
+    False,
 )
 
 # Artifacts
@@ -36,7 +36,6 @@
 
 
 def test_basic_artifact():
-    store.sink.generate_stix_schema()
     artifact_stix_object = stix2.parse(basic_artifact_dict)
     store.add(artifact_stix_object)
     read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
@@ -46,7 +45,6 @@ def test_basic_artifact():
 
 
 def test_encrypted_artifact():
-    store.sink.generate_stix_schema()
     artifact_stix_object = stix2.parse(encrypted_artifact_dict)
     store.add(artifact_stix_object)
     read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
@@ -69,7 +67,6 @@ def test_encrypted_artifact():
 
 
 def test_autonomous_system():
-    store.sink.generate_stix_schema()
     as_obj = stix2.parse(as_dict)
     store.add(as_obj)
     read_obj = json.loads(store.get(as_obj['id']).serialize())
@@ -95,7 +92,6 @@ def test_autonomous_system():
 
 
 def test_directory():
-    store.sink.generate_stix_schema()
     directory_obj = stix2.parse(directory_dict)
     store.add(directory_obj)
     read_obj = json.loads(store.get(directory_obj['id']).serialize())
@@ -119,7 +115,6 @@ def test_directory():
 
 
 def test_domain_name():
-    store.sink.generate_stix_schema()
     domain_name_obj = stix2.parse(domain_name_dict)
     store.add(domain_name_obj)
     read_obj = json.loads(store.get(domain_name_obj['id']).serialize())
@@ -140,7 +135,6 @@ def test_domain_name():
 
 
 def test_email_addr():
-    store.sink.generate_stix_schema()
     email_addr_stix_object = stix2.parse(email_addr_dict)
     store.add(email_addr_stix_object)
     read_obj = json.loads(store.get(email_addr_stix_object['id']).serialize())
@@ -226,7 +220,6 @@ def test_email_addr():
 
 
 def test_email_msg():
-    store.sink.generate_stix_schema()
     email_msg_stix_object = stix2.parse(email_msg_dict)
     store.add(email_msg_stix_object)
     read_obj = json.loads(store.get(email_msg_stix_object['id']).serialize())
@@ -244,7 +237,6 @@ def test_email_msg():
 
 
 def test_multipart_email_msg():
-    store.sink.generate_stix_schema()
     multipart_email_msg_stix_object = stix2.parse(multipart_email_msg_dict)
     store.add(multipart_email_msg_stix_object)
     read_obj = json.loads(store.get(multipart_email_msg_stix_object['id']).serialize())
@@ -286,7 +278,6 @@ def test_multipart_email_msg():
 
 
 def test_file():
-    store.sink.generate_stix_schema()
     file_stix_object = stix2.parse(file_dict)
     store.add(file_stix_object)
     read_obj = store.get(file_stix_object['id'])
@@ -318,7 +309,6 @@ def test_file():
 
 
 def test_ipv4():
-    store.sink.generate_stix_schema()
     ipv4_stix_object = stix2.parse(ipv4_dict)
     store.add(ipv4_stix_object)
     read_obj = store.get(ipv4_stix_object['id'])
@@ -328,7 +318,6 @@ def test_ipv4():
 
 
 def test_ipv6():
-    store.sink.generate_stix_schema()
     ipv6_stix_object = stix2.parse(ipv6_dict)
     store.add(ipv6_stix_object)
     read_obj = store.get(ipv6_stix_object['id'])
@@ -347,7 +336,6 @@ def test_ipv6():
 
 
 def test_mutex():
-    store.sink.generate_stix_schema()
     mutex_stix_object = stix2.parse(mutex_dict)
     store.add(mutex_stix_object)
     read_obj = store.get(mutex_stix_object['id'])
@@ -388,7 +376,6 @@ def test_mutex():
 
 
 def test_network_traffic():
-    store.sink.generate_stix_schema()
     network_traffic_stix_object = stix2.parse(network_traffic_dict)
     store.add(network_traffic_stix_object)
     read_obj = store.get(network_traffic_stix_object['id'])
@@ -429,7 +416,6 @@ def test_network_traffic():
 
 
 def test_process():
-    store.sink.generate_stix_schema()
     process_stix_object = stix2.parse(process_dict)
     store.add(process_stix_object)
     read_obj = store.get(process_stix_object['id'])
@@ -458,7 +444,6 @@ def test_process():
 
 
 def test_software():
-    store.sink.generate_stix_schema()
     software_stix_object = stix2.parse(software_dict)
     store.add(software_stix_object)
     read_obj = store.get(software_stix_object['id'])
@@ -475,16 +460,8 @@ def test_software():
     "value": "https://example.com/research/index.html",
 }
 
-store = RelationalDBStore(
-        "postgresql://postgres:admin@localhost/postgres",
-        False,
-        None,
-        True,
-)
-
 
 def test_url():
-    store.sink.generate_stix_schema()
     url_stix_object = stix2.parse(url_dict)
     store.add(url_stix_object)
     read_obj = json.loads(store.get(url_stix_object['id']).serialize())
@@ -516,7 +493,6 @@ def test_url():
 
 
 def test_user_account():
-    store.sink.generate_stix_schema()
     user_account_stix_object = stix2.parse(user_account_dict)
     store.add(user_account_stix_object)
     read_obj = json.loads(store.get(user_account_stix_object['id']).serialize())
@@ -557,7 +533,6 @@ def test_user_account():
 
 
 def test_windows_registry():
-    store.sink.generate_stix_schema()
     windows_registry_stix_object = stix2.parse(windows_registry_dict)
     store.add(windows_registry_stix_object)
     read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
@@ -618,7 +593,6 @@ def test_windows_registry():
 
 
 def test_basic_x509_certificate():
-    store.sink.generate_stix_schema()
     basic_x509_certificate_stix_object = stix2.parse(basic_x509_certificate_dict)
     store.add(basic_x509_certificate_stix_object)
     read_obj = json.loads(store.get(basic_x509_certificate_stix_object['id']).serialize())
@@ -633,7 +607,6 @@ def test_basic_x509_certificate():
 
 
 def test_x509_certificate_with_extensions():
-    store.sink.generate_stix_schema()
     extensions_x509_certificate_stix_object = stix2.parse(extensions_x509_certificate_dict)
     store.add(extensions_x509_certificate_stix_object)
     read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())

From eb21abff77f3da7c11e8c317b640a8667828ba47 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:37:27 -0400
Subject: [PATCH 087/132] add postgres to pipeline

---
 .github/workflows/python-ci-tests.yml | 26 +++++++++++++++++++++++++-
 1 file changed, 25 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 6ca27d5f..91c922ed 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -4,8 +4,32 @@ name: cti-python-stix2 test harness
 on: [push, pull_request]
 
 jobs:
-  build:
+  # Label of the container job
+  container-job:
+    # Containers must run in Linux based operating systems
+    runs-on: ubuntu-latest
+    # Docker Hub image that `container-job` executes in
+    container: node:10.18-jessie
 
+    # Service containers to run with `container-job`
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+  build:
     runs-on: ubuntu-latest
     strategy:
       matrix:

From cccf04c3a8a1cec7d4ce1cb7c6b7d58433851ae8 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:41:18 -0400
Subject: [PATCH 088/132] add postgres to pipeline, and use it

---
 .github/workflows/python-ci-tests.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 91c922ed..6bffee36 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -48,6 +48,7 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Test with Tox
+      uses: container-job
       run: |
         tox
     - name: Upload coverage information to Codecov

From a2593893b3d942fe7555faabc885166fd67e92c3 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:43:29 -0400
Subject: [PATCH 089/132] add another step for postgres

---
 .github/workflows/python-ci-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 6bffee36..3fc774b0 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -47,8 +47,9 @@ jobs:
         pip install -U pip setuptools
         pip install tox-gh-actions
         pip install codecov
-    - name: Test with Tox
+    - name: Connect to PostgreSQL
       uses: container-job
+    - name: Test with Tox
       run: |
         tox
     - name: Upload coverage information to Codecov

From 4286ef5de125dec794ee30c469fe85b9bbbcdd75 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:45:22 -0400
Subject: [PATCH 090/132] don't use the job name

---
 .github/workflows/python-ci-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 3fc774b0..5358a286 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -48,7 +48,7 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Connect to PostgreSQL
-      uses: container-job
+      uses: postgres
     - name: Test with Tox
       run: |
         tox

From 3c504ee99524387dfbbda4552514d5636cdcb728 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:53:12 -0400
Subject: [PATCH 091/132] only one job

---
 .github/workflows/python-ci-tests.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 5358a286..0ef449af 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -29,8 +29,6 @@ jobs:
         ports:
           # Maps tcp port 5432 on service container to the host
           - 5432:5432
-  build:
-    runs-on: ubuntu-latest
     strategy:
       matrix:
         python-version: [3.8, 3.9, '3.10', '3.11', '3.12']

From 38f60a2d9e1970ff5d56a2574b05a742d619ca1c Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:56:24 -0400
Subject: [PATCH 092/132] added docker:

---
 .github/workflows/python-ci-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 0ef449af..975a3eb7 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -46,7 +46,7 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Connect to PostgreSQL
-      uses: postgres
+      uses: docker:postgres
     - name: Test with Tox
       run: |
         tox

From 4abcf3da2e7fd7b4419d20b531a19634cc1c5490 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 13:57:22 -0400
Subject: [PATCH 093/132] added //

---
 .github/workflows/python-ci-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 975a3eb7..e9053444 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -46,7 +46,7 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Connect to PostgreSQL
-      uses: docker:postgres
+      uses: docker://postgres
     - name: Test with Tox
       run: |
         tox

From 68497bbd7bbf9e29e3ef8314a92b80bb3bb65b1a Mon Sep 17 00:00:00 2001
From: Robert Thew <rthew@mitre.org>
Date: Tue, 16 Apr 2024 14:21:49 -0400
Subject: [PATCH 094/132] Updated to add_method to check class names

---
 stix2/datastore/relational_db/add_method.py    | 17 +++++++++++------
 stix2/datastore/relational_db/utils.py         | 14 +++++++-------
 stix2/test/v21/test_datastore_relational_db.py |  4 +---
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/stix2/datastore/relational_db/add_method.py b/stix2/datastore/relational_db/add_method.py
index 02d10c80..8035983f 100644
--- a/stix2/datastore/relational_db/add_method.py
+++ b/stix2/datastore/relational_db/add_method.py
@@ -1,19 +1,24 @@
 import re
+from stix2.v21.base import _STIXBase21
+from stix2.datastore.relational_db.utils import get_all_subclasses
+from stix2.properties import Property
 
-# _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
-#
-#
-# _ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
+_ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
+_ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
+_ALLOWABLE_CLASSES.extend([Property])
 
 
 def create_real_method_name(name, klass_name):
-    # if klass_name not in _ALLOWABLE_CLASSES:
-    #     raise NameError
+    classnames = map(lambda x: x.__name__, _ALLOWABLE_CLASSES)
+    if klass_name not in classnames:
+        raise NameError
+
     split_up_klass_name = re.findall('[A-Z][^A-Z]*', klass_name)
     return name + "_" + "_".join([x.lower() for x in split_up_klass_name])
 
 
 def add_method(cls):
+
     def decorator(fn):
         method_name = fn.__name__
         fn.__name__ = create_real_method_name(fn.__name__, cls.__name__)
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 0958958f..0ae0bf4e 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -52,22 +52,22 @@ def canonicalize_table_name(table_name, schema_name=None):
     return inflection.underscore(full_name)
 
 
-def _get_all_subclasses(cls):
+def get_all_subclasses(cls):
     all_subclasses = []
 
     for subclass in cls.__subclasses__():
         all_subclasses.append(subclass)
-        all_subclasses.extend(_get_all_subclasses(subclass))
+        all_subclasses.extend(get_all_subclasses(subclass))
     return all_subclasses
 
 
 def get_stix_object_classes():
-    yield from _get_all_subclasses(_DomainObject)
-    yield from _get_all_subclasses(_RelationshipObject)
-    yield from _get_all_subclasses(_Observable)
-    yield from _get_all_subclasses(_MetaObject)
+    yield from get_all_subclasses(_DomainObject)
+    yield from get_all_subclasses(_RelationshipObject)
+    yield from get_all_subclasses(_Observable)
+    yield from get_all_subclasses(_MetaObject)
     # Non-object extensions (property or toplevel-property only)
-    for ext_cls in _get_all_subclasses(_Extension):
+    for ext_cls in get_all_subclasses(_Extension):
         if ext_cls.extension_type not in (
             "new-sdo", "new-sco", "new-sro",
         ):
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index f9626dcd..a4061abb 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -9,6 +9,7 @@
     True,
     None,
     False,
+    False
 )
 
 # Artifacts
@@ -280,7 +281,6 @@ def test_multipart_email_msg():
 def test_file():
     file_stix_object = stix2.parse(file_dict)
     store.add(file_stix_object)
-    read_obj = store.get(file_stix_object['id'])
     read_obj = json.loads(store.get(file_stix_object['id']).serialize())
 
     for attrib in file_dict.keys():
@@ -418,7 +418,6 @@ def test_network_traffic():
 def test_process():
     process_stix_object = stix2.parse(process_dict)
     store.add(process_stix_object)
-    read_obj = store.get(process_stix_object['id'])
     read_obj = json.loads(store.get(process_stix_object['id']).serialize())
 
     for attrib in process_dict.keys():
@@ -446,7 +445,6 @@ def test_process():
 def test_software():
     software_stix_object = stix2.parse(software_dict)
     store.add(software_stix_object)
-    read_obj = store.get(software_stix_object['id'])
     read_obj = json.loads(store.get(software_stix_object['id']).serialize())
 
     for attrib in software_dict.keys():

From dbe31d490dad2c394eddb6f284d890395e814c6c Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 15:37:25 -0400
Subject: [PATCH 095/132] slightly different

---
 .github/workflows/python-ci-tests.yml | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 7ea2ee01..a6b0b0de 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -4,31 +4,25 @@ name: cti-python-stix2 test harness
 on: [push, pull_request]
 
 jobs:
-  # Label of the container job
-  container-job:
-    # Containers must run in Linux based operating systems
+  test-job:
     runs-on: ubuntu-latest
-    # Docker Hub image that `container-job` executes in
-    container: node:10.18-jessie
 
-    # Service containers to run with `container-job`
     services:
-      # Label used to access the service container
       postgres:
-        # Docker Hub image
-        image: postgres
+        image: postgres:11
         # Provide the password for postgres
         env:
+          POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports: [ '5432:5432' ]
         # Set health checks to wait until postgres has started
         options: >-
           --health-cmd pg_isready
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
-        ports:
-          # Maps tcp port 5432 on service container to the host
-          - 5432:5432
+
     strategy:
       matrix:
         python-version: [3.8, 3.9, '3.10', '3.11', '3.12']
@@ -45,8 +39,6 @@ jobs:
         pip install -U pip setuptools
         pip install tox-gh-actions
         pip install codecov
-    - name: Connect to PostgreSQL
-      uses: docker://postgres
     - name: Test with Tox
       run: |
         tox

From 1583e4de370e98c57585bc3ed6e0cdb02c12995c Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 15:43:05 -0400
Subject: [PATCH 096/132] change postgres url

---
 stix2/test/v21/test_datastore_relational_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index a4061abb..ffdd4a19 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -5,7 +5,7 @@
 import stix2.properties
 
 store = RelationalDBStore(
-    "postgresql://localhost/stix-data-sink",
+    "postgresql://localhost/postgres",
     True,
     None,
     False,

From 2d578e3b46158bb39bc3c365897da28f1ff8d1df Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 15:55:20 -0400
Subject: [PATCH 097/132] different url, and no create_tables or metadata in
 store

---
 stix2/datastore/relational_db/relational_db.py | 10 +++++-----
 stix2/test/v21/test_datastore_relational_db.py |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 6f23984c..cc8e9629 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -78,11 +78,11 @@ def __init__(
                 them.
         """
         database_connection = create_engine(database_connection_url)
-
-        self.metadata = MetaData()
-        create_table_objects(
-            self.metadata, stix_object_classes,
-        )
+        print(database_connection)
+        # self.metadata = MetaData()
+        # create_table_objects(
+        #     self.metadata, stix_object_classes,
+        # )
 
         super().__init__(
             source=RelationalDBSource(
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index ffdd4a19..6e8f33da 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -5,7 +5,7 @@
 import stix2.properties
 
 store = RelationalDBStore(
-    "postgresql://localhost/postgres",
+    "postgresql://Postgres:5432/postgres",
     True,
     None,
     False,

From b54166a501276e46bfdfa1c7c151cb95e5a429f1 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 15:58:35 -0400
Subject: [PATCH 098/132] create the metadata after all

---
 stix2/datastore/relational_db/relational_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index cc8e9629..a2ed447e 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -79,7 +79,7 @@ def __init__(
         """
         database_connection = create_engine(database_connection_url)
         print(database_connection)
-        # self.metadata = MetaData()
+        self.metadata = MetaData()
         # create_table_objects(
         #     self.metadata, stix_object_classes,
         # )

From 203fc26adfd58be4a089589ee04b8c87f9a2a393 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:01:09 -0400
Subject: [PATCH 099/132] use 0.0.0.0

---
 stix2/test/v21/test_datastore_relational_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 6e8f33da..770d284c 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -5,7 +5,7 @@
 import stix2.properties
 
 store = RelationalDBStore(
-    "postgresql://Postgres:5432/postgres",
+    "postgresql://0.0.0.0:5432/postgres",
     True,
     None,
     False,

From 585e6df0996b65df9594f77b2114af278285e59f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:11:54 -0400
Subject: [PATCH 100/132] use env vars in postgres url

---
 stix2/test/v21/test_datastore_relational_db.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 770d284c..0a8751b4 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -1,11 +1,12 @@
 import json
+import os
 
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
 store = RelationalDBStore(
-    "postgresql://0.0.0.0:5432/postgres",
+    f"postgresql://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}@0.0.0.0:5432/postgres",
     True,
     None,
     False,

From 11cef1bbf8f0cc139b55f1ea4ad057dd9fc3e82a Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:15:52 -0400
Subject: [PATCH 101/132] add env vars to tox run

---
 .github/workflows/python-ci-tests.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index a6b0b0de..066689a2 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -40,6 +40,10 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Test with Tox
+      env:
+        POSTGRES_USER: postgres
+        POSTGRES_PASSWORD: postgres
+        POSTGRES_DB: postgres
       run: |
         tox
     - name: Upload coverage information to Codecov

From 8ca924afb292e4069e3b14f9890f16bea1bf00ae Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:23:30 -0400
Subject: [PATCH 102/132] add env vars at top of workflow

---
 .github/workflows/python-ci-tests.yml | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/python-ci-tests.yml b/.github/workflows/python-ci-tests.yml
index 066689a2..2dc51d85 100644
--- a/.github/workflows/python-ci-tests.yml
+++ b/.github/workflows/python-ci-tests.yml
@@ -3,6 +3,11 @@
 name: cti-python-stix2 test harness
 on: [push, pull_request]
 
+env:
+  POSTGRES_USER: postgres
+  POSTGRES_PASSWORD: postgres
+  POSTGRES_DB: postgres
+
 jobs:
   test-job:
     runs-on: ubuntu-latest
@@ -40,10 +45,6 @@ jobs:
         pip install tox-gh-actions
         pip install codecov
     - name: Test with Tox
-      env:
-        POSTGRES_USER: postgres
-        POSTGRES_PASSWORD: postgres
-        POSTGRES_DB: postgres
       run: |
         tox
     - name: Upload coverage information to Codecov

From d52bf2d1892a4b52e1dd1227c237b0fb9cb58d71 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:30:03 -0400
Subject: [PATCH 103/132] default values for the env vars

---
 stix2/test/v21/test_datastore_relational_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 0a8751b4..c9a774f3 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -6,7 +6,7 @@
 import stix2.properties
 
 store = RelationalDBStore(
-    f"postgresql://{os.environ['POSTGRES_USER']}:{os.environ['POSTGRES_PASSWORD']}@0.0.0.0:5432/postgres",
+    f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
     True,
     None,
     False,

From dc69b75599a2333153e5b897ba213d20fc0816df Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:33:53 -0400
Subject: [PATCH 104/132] instantiate the db

---
 stix2/test/v21/test_datastore_relational_db.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index c9a774f3..a48dd6c5 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -9,8 +9,6 @@
     f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
     True,
     None,
-    False,
-    False
 )
 
 # Artifacts

From d15be9644d74cc0579e05eac2cb46c8b24d94db2 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 16 Apr 2024 16:47:51 -0400
Subject: [PATCH 105/132] create the tables in sqlalchemy

---
 stix2/datastore/relational_db/relational_db.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index a2ed447e..1a726824 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -80,9 +80,9 @@ def __init__(
         database_connection = create_engine(database_connection_url)
         print(database_connection)
         self.metadata = MetaData()
-        # create_table_objects(
-        #     self.metadata, stix_object_classes,
-        # )
+        create_table_objects(
+             self.metadata, stix_object_classes,
+        )
 
         super().__init__(
             source=RelationalDBSource(

From 4067ba753d828182a6521824ce0f595bbb711a25 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 21 Aug 2024 15:44:37 -0400
Subject: [PATCH 106/132] fixed kill_chains, granular_markings,
 extension-definitions

---
 stix2/datastore/relational_db/add_method.py   |  3 +-
 .../datastore/relational_db/input_creation.py | 83 +++++++++++++------
 .../datastore/relational_db/relational_db.py  | 17 +++-
 .../relational_db/relational_db_testing.py    | 75 ++++++++++++++++-
 .../datastore/relational_db/table_creation.py |  5 +-
 5 files changed, 153 insertions(+), 30 deletions(-)

diff --git a/stix2/datastore/relational_db/add_method.py b/stix2/datastore/relational_db/add_method.py
index 8035983f..61d4b3ff 100644
--- a/stix2/datastore/relational_db/add_method.py
+++ b/stix2/datastore/relational_db/add_method.py
@@ -1,7 +1,8 @@
 import re
-from stix2.v21.base import _STIXBase21
+
 from stix2.datastore.relational_db.utils import get_all_subclasses
 from stix2.properties import Property
+from stix2.v21.base import _STIXBase21
 
 _ALLOWABLE_CLASSES = get_all_subclasses(_STIXBase21)
 _ALLOWABLE_CLASSES.extend(get_all_subclasses(Property))
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index ee0a0a3c..3b8bb23c 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -11,6 +11,8 @@
     HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
     Property, ReferenceProperty, StringProperty, TimestampProperty,
 )
+from stix2.utils import STIXdatetime
+from stix2.v21.common import KillChainPhase
 
 
 @add_method(Property)
@@ -29,7 +31,7 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(DictionaryProperty)
-def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
+def generate_insert_information(self, dictionary_name, stix_object, **kwargs):  # noqa: F811
     bindings = dict()
     data_sink = kwargs.get("data_sink")
     table_name = kwargs.get("table_name")
@@ -42,21 +44,29 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
         bindings["id"] = foreign_key_value
     table = data_sink.tables_dictionary[
         canonicalize_table_name(
-            table_name + "_" + name,
+            table_name + "_" + dictionary_name,
             schema_name,
         )
     ]
-    for name, value in stix_object[name].items():
 
-        name_binding = "name"
-        if not hasattr(self, "value_types") or len(self.value_types) == 1:
+    # binary, boolean, float, hex,
+    # integer, string, timestamp
+    valid_types = stix_object._properties[dictionary_name].valid_types
+    for name, value in stix_object[dictionary_name].items():
+        if not valid_types or len(self.valid_types) == 1:
             value_binding = "value"
-        elif isinstance(value, int):
+        elif isinstance(value, int) and IntegerProperty in valid_types:
             value_binding = "integer_value"
+        elif isinstance(value, str) and StringProperty in valid_types:
+            value_binding = "string_value"
+        elif isinstance(value, bool) and BooleanProperty in valid_types:
+            value_binding = "boolean_value"
+        elif isinstance(value, STIXdatetime) and TimestampProperty in valid_types:
+            value_binding = "timestamp_value"
         else:
             value_binding = "string_value"
 
-        bindings[name_binding] = name
+        bindings["name"] = name
         bindings[value_binding] = value
 
         insert_statements.append(insert(table).values(bindings))
@@ -175,6 +185,29 @@ def generate_insert_information(   # noqa: F811
             }
             insert_statements.append(insert(table).values(bindings))
         return insert_statements
+    elif self.contained == KillChainPhase:
+        insert_statements = list()
+        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
+
+        for idx, item in enumerate(stix_object[name]):
+            bindings = {
+                "id": stix_object["id"] if id in stix_object else foreign_key_value,
+                "kill_chain_name": item["kill_chain_name"],
+                "phase_name": item["phase_name"],
+            }
+            insert_statements.append(insert(table).values(bindings))
+        return insert_statements
+    elif isinstance(self.contained, EnumProperty):
+        insert_statements = list()
+        table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
+
+        for idx, item in enumerate(stix_object[name]):
+            bindings = {
+                "id": stix_object["id"] if id in stix_object else foreign_key_value,
+                name: item,
+            }
+            insert_statements.append(insert(table).values(bindings))
+        return insert_statements
     elif isinstance(self.contained, EmbeddedObjectProperty):
         insert_statements = list()
         for value in stix_object[name]:
@@ -265,20 +298,19 @@ def generate_insert_for_external_references(data_sink, stix_object):
 
 
 def generate_insert_for_granular_markings(granular_markings_table, stix_object):
+    insert_statements = list()
     granular_markings = stix_object["granular_markings"]
-    bindings = {
-        "id": stix_object["id"],
-    }
     for idx, granular_marking in enumerate(granular_markings):
-        lang_binding_name = f"lang{idx}"
-        marking_ref_binding_name = f"marking_ref{idx}"
-        selectors_binding_name = f"selectors{idx}"
-
-        bindings[lang_binding_name] = granular_marking.get("lang")
-        bindings[marking_ref_binding_name] = granular_marking.get("marking_ref")
-        bindings[selectors_binding_name] = granular_marking.get("selectors")
-
-    return [insert(granular_markings_table).values(bindings)]
+        bindings = {"id": stix_object["id"]}
+        lang_property_value = granular_marking.get("lang")
+        if lang_property_value:
+            bindings["lang"] = lang_property_value
+        marking_ref_value = granular_marking.get("marking_ref")
+        if marking_ref_value:
+            bindings["marking_ref"] = marking_ref_value
+        bindings["selectors"] = granular_marking.get("selectors")
+        insert_statements.append(insert(granular_markings_table).values(bindings))
+    return insert_statements
 
 
 # def generate_insert_for_extensions(extensions, foreign_key_value, type_name, core_properties):
@@ -297,7 +329,7 @@ def generate_insert_for_granular_markings(granular_markings_table, stix_object):
 
 
 def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
-    if schema_name == "sdo":
+    if schema_name in ["sdo", "sro"]:
         core_table = data_sink.tables_dictionary["common.core_sdo"]
     else:
         core_table = data_sink.tables_dictionary["common.core_sco"]
@@ -335,7 +367,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
             granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
         granular_input_statements = generate_insert_for_granular_markings(
             granular_marking_table,
-            stix_object.granular_markings,
+            stix_object,
         )
         insert_statements.extend(granular_input_statements)
 
@@ -393,17 +425,20 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
     bindings = dict()
     if schema_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES
-    else:
+    elif schema_name in ["sdo", "sro"]:
         core_properties = SDO_COMMON_PROPERTIES
+    else:
+        core_properties = list()
     type_name = stix_object["type"]
-    insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
+    if core_properties:
+        insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
     if "id" in stix_object:
         foreign_key_value = stix_object["id"]
     else:
         foreign_key_value = None
     sub_insert_statements = list()
     for name, prop in stix_object._properties.items():
-        if (name == 'id' or name not in core_properties) and name in stix_object:
+        if (name == 'id' or name not in core_properties) and name != "type" and name in stix_object:
             result = prop.generate_insert_information(
                 name, stix_object,
                 data_sink=data_sink,
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 1a726824..434e37f8 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -2,7 +2,9 @@
 from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
-from stix2.base import _STIXBase
+from stix2.base import (
+    _DomainObject, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
+)
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.relational_db.input_creation import (
     generate_insert_for_object,
@@ -187,8 +189,19 @@ def add(self, stix_data, version=None):
         _add(self, stix_data)
     add.__doc__ = _add.__doc__
 
+    @staticmethod
+    def _determine_schema_name(stix_object):
+        if isinstance(stix_object, _DomainObject):
+            return "sdo"
+        elif isinstance(stix_object, _Observable):
+            return "sco"
+        elif isinstance(stix_object, _RelationshipObject):
+            return "sro"
+        elif isinstance(stix_object, _MetaObject):
+            return "common"
+
     def insert_object(self, stix_object):
-        schema_name = "sdo" if "created" in stix_object else "sco"
+        schema_name = self._determine_schema_name(stix_object)
         with self.database_connection.begin() as trans:
             statements = generate_insert_for_object(self, stix_object, schema_name)
             for stmt in statements:
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 853d5d9f..89245748 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -77,6 +77,7 @@ def malware_with_all_required_properties():
 def file_example_with_PDFExt_Object():
     f = stix2.v21.File(
         name="qwerty.dll",
+        magic_number_hex="504B0304",
         extensions={
             "pdf-ext": stix2.v21.PDFExt(
                 version="1.7",
@@ -95,6 +96,66 @@ def file_example_with_PDFExt_Object():
     return f
 
 
+def extension_definition_insert():
+    return stix2.ExtensionDefinition(
+        created_by_ref="identity--8a5fb7e4-aabe-4635-8972-cbcde1fa4792",
+        name="test",
+        schema="a schema",
+        version="1.2.3",
+        extension_types=["property-extension", "new-sdo", "new-sro"],
+    )
+
+
+def dictionary_test():
+    return stix2.File(
+        spec_version="2.1",
+        name="picture.jpg",
+        defanged=True,
+        ctime="1980-02-23T05:43:28.2678Z",
+        extensions={
+          "raster-image-ext": {
+              "exif_tags": {
+                  "Make": "Nikon",
+                  "Model": "D7000",
+                  "XResolution": 4928,
+                  "YResolution": 3264,
+              },
+          },
+        },
+    )
+
+
+def kill_chain_test():
+    return stix2.AttackPattern(
+        spec_version="2.1",
+        id="attack-pattern--0c7b5b88-8ff7-4a4d-aa9d-feb398cd0061",
+        created="2016-05-12T08:17:27.000Z",
+        modified="2016-05-12T08:17:27.000Z",
+        name="Spear Phishing",
+        kill_chain_phases=[
+             {
+                 "kill_chain_name": "lockheed-martin-cyber-kill-chain",
+                 "phase_name": "reconnaissance",
+             },
+        ],
+        external_references=[
+             {
+                 "source_name": "capec",
+                 "external_id": "CAPEC-163",
+             },
+        ],
+        granular_markings=[
+             {
+                 "lang": "en_US",
+                 "selectors": ["kill_chain_phases"],
+             },
+             {
+                 "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
+                 "selectors": ["external_references"],
+             },
+        ], )
+
+
 def main():
     store = RelationalDBStore(
         "postgresql://localhost/stix-data-sink",
@@ -102,15 +163,27 @@ def main():
         None,
         True,
         False,
-        stix2.Directory,
     )
 
     if store.sink.database_exists:
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 
+        pdf_file = file_example_with_PDFExt_Object()
+        store.add(pdf_file)
+
+        ap = kill_chain_test()
+        store.add(ap)
+
         store.add(directory_stix_object)
 
+        store.add(s)
+
+        store.add(extension_definition_insert())
+
+        dict_example = dictionary_test()
+        store.add(dict_example)
+
         read_obj = store.get(directory_stix_object.id)
         print(read_obj)
     else:
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index ee84e138..3318229f 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -587,7 +587,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
         )
         return tables
     elif self.contained == KillChainPhase:
-        tables.append(create_kill_chain_phases_table("kill_chain_phase", metadata, schema_name, table_name))
+        tables.append(create_kill_chain_phases_table("kill_chain_phases", metadata, schema_name, table_name))
         return tables
     else:
         if isinstance(self.contained, Property):
@@ -693,7 +693,8 @@ def generate_object_table(
     columns = list()
     tables = list()
     for name, prop in properties.items():
-        if name == 'id' or name not in core_properties:
+        # type is never a column since it is implicit in the table
+        if (name == 'id' or name not in core_properties) and name != 'type':
             col = prop.generate_table_information(
                 name,
                 metadata=metadata,

From 7d7cf7a84190e90f1c1dfdd20479a4c11b54ec38 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 21 Aug 2024 16:24:08 -0400
Subject: [PATCH 107/132] try to fix test_datastore_relational_db

---
 stix2/test/v21/test_datastore_relational_db.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index a48dd6c5..39b461e6 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -9,6 +9,7 @@
     f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
     True,
     None,
+    False
 )
 
 # Artifacts

From 675746bc280bc529fde26104eb695c9cbdb1e1cb Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 10 Sep 2024 14:26:35 -0400
Subject: [PATCH 108/132] fix core properties for SMOs and remove hardcoded
 name for kill chain tables

---
 .../datastore/relational_db/input_creation.py |  6 ++--
 .../relational_db/relational_db_testing.py    | 36 +++++++++++++++++++
 .../datastore/relational_db/table_creation.py |  2 +-
 3 files changed, 40 insertions(+), 4 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 3b8bb23c..23cd56e6 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -329,7 +329,7 @@ def generate_insert_for_granular_markings(granular_markings_table, stix_object):
 
 
 def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
-    if schema_name in ["sdo", "sro"]:
+    if schema_name in ["sdo", "sro", "common"]:
         core_table = data_sink.tables_dictionary["common.core_sdo"]
     else:
         core_table = data_sink.tables_dictionary["common.core_sco"]
@@ -361,7 +361,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
 
     # Granular markings
     if "granular_markings" in stix_object:
-        if schema_name == "sdo":
+        if schema_name != "sco":
             granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
         else:
             granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
@@ -425,7 +425,7 @@ def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
     bindings = dict()
     if schema_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES
-    elif schema_name in ["sdo", "sro"]:
+    elif schema_name in ["sdo", "sro", "common"]:
         core_properties = SDO_COMMON_PROPERTIES
     else:
         core_properties = list()
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 89245748..540b5eae 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -103,6 +103,16 @@ def extension_definition_insert():
         schema="a schema",
         version="1.2.3",
         extension_types=["property-extension", "new-sdo", "new-sro"],
+        granular_markings=[
+            {
+                "lang": "en_US",
+                "selectors": ["name", "schema"]
+            },
+            {
+                "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
+                "selectors": ["name", "schema"]
+            }
+        ]
     )
 
 
@@ -155,6 +165,28 @@ def kill_chain_test():
              },
         ], )
 
+@stix2.CustomObject('x-custom-type',
+        properties=[
+            ("phases", stix2.properties.ListProperty(stix2.KillChainPhase)),
+            ("test", stix2.properties.IntegerProperty())
+        ]
+    )
+class CustomClass:
+    pass
+
+
+def custom_obj():
+    obj = CustomClass(
+        phases=[
+            {
+                "kill_chain_name": "chain name",
+                "phase_name": "the phase name"
+            }
+        ],
+        test=5
+    )
+    return obj
+
 
 def main():
     store = RelationalDBStore(
@@ -169,6 +201,10 @@ def main():
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 
+        co = custom_obj()
+
+        store.add(co)
+
         pdf_file = file_example_with_PDFExt_Object()
         store.add(pdf_file)
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 3318229f..d7fd621c 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -587,7 +587,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
         )
         return tables
     elif self.contained == KillChainPhase:
-        tables.append(create_kill_chain_phases_table("kill_chain_phases", metadata, schema_name, table_name))
+        tables.append(create_kill_chain_phases_table(name, metadata, schema_name, table_name))
         return tables
     else:
         if isinstance(self.contained, Property):

From aa8c9a11f946d6300aa3e60029ea2136519e539f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 10 Sep 2024 19:25:33 -0400
Subject: [PATCH 109/132] handle object-markings-refs in SMOs

---
 stix2/datastore/relational_db/input_creation.py        | 2 +-
 stix2/datastore/relational_db/relational_db_testing.py | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 23cd56e6..1b859751 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -347,7 +347,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
     insert_statements.append(core_insert_statement)
 
     if "object_marking_refs" in stix_object:
-        if schema_name == "sdo":
+        if schema_name != "sco":
             object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
         else:
             object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 540b5eae..b051dc86 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -103,6 +103,10 @@ def extension_definition_insert():
         schema="a schema",
         version="1.2.3",
         extension_types=["property-extension", "new-sdo", "new-sro"],
+        object_marking_refs=[
+            "marking-definition--caa0d913-5db8-4424-aae0-43e770287d30",
+            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77"
+        ],
         granular_markings=[
             {
                 "lang": "en_US",
@@ -168,7 +172,7 @@ def kill_chain_test():
 @stix2.CustomObject('x-custom-type',
         properties=[
             ("phases", stix2.properties.ListProperty(stix2.KillChainPhase)),
-            ("test", stix2.properties.IntegerProperty())
+            ("something_else", stix2.properties.IntegerProperty())
         ]
     )
 class CustomClass:
@@ -183,7 +187,7 @@ def custom_obj():
                 "phase_name": "the phase name"
             }
         ],
-        test=5
+        something_else=5
     )
     return obj
 

From 27fbfa3ebc7536d96411a49055abc9f536b7705e Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 10 Sep 2024 19:47:54 -0400
Subject: [PATCH 110/132] drop db when testing

---
 stix2/datastore/relational_db/relational_db_testing.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index b051dc86..699e2262 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -198,7 +198,7 @@ def main():
         False,
         None,
         True,
-        False,
+        True,
     )
 
     if store.sink.database_exists:

From 9814e6cbe42706bc49e296db3471c1657b3c5bd1 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 12 Sep 2024 21:16:02 -0400
Subject: [PATCH 111/132] removed extension-definition object_ref table, added
 fk to SMOs

---
 .../datastore/relational_db/table_creation.py | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index d7fd621c..b3cb4ac1 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -470,34 +470,34 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 @add_method(IDProperty)
 def generate_table_information(self, name, **kwargs):  # noqa: F811
     schema_name = kwargs.get('schema_name')
-    if schema_name == "sro":
-        # sro common properties are the same as sdo's
+    if schema_name in ["sro", "common"]:
+        # sro, smo common properties are the same as sdo's
         schema_name = "sdo"
     table_name = kwargs.get("table_name")
-    if schema_name == "common":
-        return Column(
-            name,
-            Text,
-            CheckConstraint(
-                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-            nullable=not (self.required),
-        )
-    else:
-        foreign_key_column = f"common.core_{schema_name}.id"
-        return Column(
-            name,
-            Text,
-            ForeignKey(foreign_key_column, ondelete="CASCADE"),
-            CheckConstraint(
-                f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
-                # noqa: E131
-            ),
-            primary_key=True,
-            nullable=not (self.required),
-        )
+    # if schema_name == "common":
+    #     return Column(
+    #         name,
+    #         Text,
+    #         CheckConstraint(
+    #             f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+    #             # noqa: E131
+    #         ),
+    #         primary_key=True,
+    #         nullable=not (self.required),
+    #     )
+    # else:
+    foreign_key_column = f"common.core_{schema_name}.id"
+    return Column(
+        name,
+        Text,
+        ForeignKey(foreign_key_column, ondelete="CASCADE"),
+        CheckConstraint(
+            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+            # noqa: E131
+        ),
+        primary_key=True,
+        nullable=not (self.required),
+    )
 
     return Column(
         name,
@@ -684,7 +684,7 @@ def generate_object_table(
         table_name = parent_table_name + "_" + table_name
     if is_embedded_object:
         core_properties = list()
-    elif schema_name in ["sdo", "sro"]:
+    elif schema_name in ["sdo", "sro", "common"]:
         core_properties = SDO_COMMON_PROPERTIES
     elif schema_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES

From f4f1828a42088e25b2d8a3f322aa6e4afed71f86 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 12 Sep 2024 22:38:26 -0400
Subject: [PATCH 112/132] fixed binary list, dictionary insert

---
 .../datastore/relational_db/input_creation.py |  3 +
 .../relational_db/relational_db_testing.py    | 71 +++++++++++++++++++
 .../datastore/relational_db/table_creation.py |  2 +-
 3 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 1b859751..d66fd4f3 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -53,6 +53,7 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
     # integer, string, timestamp
     valid_types = stix_object._properties[dictionary_name].valid_types
     for name, value in stix_object[dictionary_name].items():
+        bindings = dict()
         if not valid_types or len(self.valid_types) == 1:
             value_binding = "value"
         elif isinstance(value, int) and IntegerProperty in valid_types:
@@ -61,6 +62,8 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
             value_binding = "string_value"
         elif isinstance(value, bool) and BooleanProperty in valid_types:
             value_binding = "boolean_value"
+        elif isinstance(value, float) and FloatProperty in valid_types:
+            value_binding = "float_value"
         elif isinstance(value, STIXdatetime) and TimestampProperty in valid_types:
             value_binding = "timestamp_value"
         else:
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 699e2262..3e720ace 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -192,6 +192,63 @@ def custom_obj():
     return obj
 
 
+@stix2.CustomObject(
+    "test-object", [
+        ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
+    ],
+    "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
+    is_sdo=True
+)
+class TestClass:
+    pass
+
+
+def test_binary_list():
+    return TestClass(prop_name=["AREi", "7t3M"])
+
+@stix2.CustomObject(
+        "test2-object", [
+            ("prop_name", stix2.properties.ListProperty(
+                stix2.properties.HexProperty()
+            ))
+        ],
+        "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
+        is_sdo=True
+    )
+
+class Test2Class:
+        pass
+
+def test_hex_list():
+    return Test2Class(
+        prop_name=["1122", "fedc"]
+    )
+
+@stix2.CustomObject(
+        "test3-object", [
+            ("prop_name",
+                 stix2.properties.DictionaryProperty(
+                     valid_types=[
+                         stix2.properties.IntegerProperty,
+                         stix2.properties.FloatProperty,
+                         stix2.properties.StringProperty
+                     ]
+                 )
+             )
+        ],
+        "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
+        is_sdo=True
+    )
+class Test3Class:
+    pass
+
+
+def test_dictionary():
+    return Test3Class(
+        prop_name={"a": 1, "b": 2.3, "c": "foo"}
+    )
+
+
 def main():
     store = RelationalDBStore(
         "postgresql://localhost/stix-data-sink",
@@ -205,6 +262,20 @@ def main():
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 
+        td = test_dictionary()
+
+        store.add(td)
+
+        th = test_hex_list()
+
+        # store.add(th)
+
+        tb = test_binary_list()
+
+        store.add(tb)
+
+
+
         co = custom_obj()
 
         store.add(co)
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index b3cb4ac1..79e61397 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -245,7 +245,7 @@ def determine_sql_type(self):  # noqa: F811
 
 @add_method(BinaryProperty)
 def determine_sql_type(self):  # noqa: F811
-    return Boolean
+    return Text
 
 
 @add_method(BooleanProperty)

From 8811fa9fe1e27aac91a3ace1d772bb4903d20e66 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Fri, 13 Sep 2024 12:35:12 -0400
Subject: [PATCH 113/132] Expand relational data store query capability; expand
 relational data store unit test suite

---
 .../datastore/relational_db/input_creation.py |   2 +-
 stix2/datastore/relational_db/query.py        | 587 ++++++++++++++++++
 .../datastore/relational_db/relational_db.py  |  46 +-
 stix2/properties.py                           |   4 +-
 .../test/v21/test_datastore_relational_db.py  | 390 +++++++++++-
 5 files changed, 964 insertions(+), 65 deletions(-)
 create mode 100644 stix2/datastore/relational_db/query.py

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index d66fd4f3..006b4fed 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -129,7 +129,7 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 @add_method(HexProperty)
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
-    v = bytes(stix_object[name], 'utf-8')
+    v = bytes.fromhex(stix_object[name])
     return {name: v}
 
 
diff --git a/stix2/datastore/relational_db/query.py b/stix2/datastore/relational_db/query.py
new file mode 100644
index 00000000..50bac5e0
--- /dev/null
+++ b/stix2/datastore/relational_db/query.py
@@ -0,0 +1,587 @@
+import inspect
+
+import sqlalchemy as sa
+
+import stix2
+from stix2.datastore import DataSourceError
+from stix2.datastore.relational_db.utils import (
+    canonicalize_table_name, schema_for, table_name_for,
+)
+import stix2.properties
+import stix2.utils
+
+
+def _check_support(stix_id):
+    """
+    Misc support checks for the relational data source.  May be better to error
+    out up front and say a type is not supported, than die with some cryptic
+    SQLAlchemy or other error later.  This runs for side-effects (raises
+    an exception) and doesn't return anything.
+
+    :param stix_id: A STIX ID.  The basis for reading an object, used to
+        determine support
+    """
+    # language-content has a complicated structure in its "contents"
+    # property, which is not currently supported for storage in a
+    # relational database.
+    stix_type = stix2.utils.get_type_from_id(stix_id)
+    if stix_type in ("language-content",):
+        raise DataSourceError(f"Reading {stix_type} objects is not supported.")
+
+
+def _tables_for(stix_class, metadata):
+    """
+    Get the core and type-specific tables for the given class
+
+    :param stix_class: A class for a STIX object type
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :return: A (core_table, type_table) 2-tuple as SQLAlchemy Table objects
+    """
+    # Info about the type-specific table
+    type_table_name = table_name_for(stix_class)
+    type_schema_name = schema_for(stix_class)
+    type_table = metadata.tables[f"{type_schema_name}.{type_table_name}"]
+
+    # Some fixed info about core tables
+    if type_schema_name == "sco":
+        core_table_name = "common.core_sco"
+    else:
+        # for SROs and SMOs too?
+        core_table_name = "common.core_sdo"
+
+    core_table = metadata.tables[core_table_name]
+
+    return core_table, type_table
+
+
+def _stix2_class_for(stix_id):
+    """
+    Find the class for the STIX type indicated by the given STIX ID.
+
+    :param stix_id: A STIX ID
+    """
+    stix_type = stix2.utils.get_type_from_id(stix_id)
+    stix_class = stix2.registry.class_for_type(
+        # TODO: give user control over STIX version used?
+        stix_type, stix_version=stix2.DEFAULT_VERSION,
+    )
+
+    return stix_class
+
+
+def _read_simple_properties(stix_id, core_table, type_table, conn):
+    """
+    Read "simple" property values, i.e. those which don't need tables other
+    than the core/type-specific tables: they're stored directly in columns of
+    those tables.  These two tables are joined and must have a defined foreign
+    key constraint between them.
+
+    :param stix_id: A STIX ID
+    :param core_table: A core table
+    :param type_table: A type-specific table
+    :param conn: An SQLAlchemy DB connection
+    :return: A mapping containing the properties and values read
+    """
+    # Both core and type-specific tables have "id"; let's not duplicate that
+    # in the result set columns.  Is there a better way to do this?
+    type_cols_except_id = (
+        col for col in type_table.c if col.key != "id"
+    )
+
+    core_type_select = sa.select(core_table, *type_cols_except_id) \
+        .join(type_table) \
+        .where(core_table.c.id == stix_id)
+
+    # Should be at most one matching row
+    obj_dict = conn.execute(core_type_select).mappings().first()
+
+    return obj_dict
+
+
+def _read_hashes(fk_id, hashes_table, conn):
+    """
+    Read hashes from a table.
+
+    :param fk_id: A foreign key value used to filter table rows
+    :param hashes_table: An SQLAlchemy Table object
+    :param conn: An SQLAlchemy DB connection
+    :return: The hashes as a dict, or None if no hashes were found
+    """
+    stmt = sa.select(hashes_table.c.hash_name, hashes_table.c.hash_value).where(
+        hashes_table.c.id == fk_id
+    )
+
+    results = conn.execute(stmt)
+    hashes = dict(results.all()) or None
+    return hashes
+
+
+def _read_external_references(stix_id, metadata, conn):
+    """
+    Read external references from some fixed tables in the common schema.
+
+    :param stix_id: A STIX ID used to filter table rows
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: The external references, as a list of dicts
+    """
+    ext_refs_table = metadata.tables["common.external_references"]
+    ext_refs_hashes_table = metadata.tables["common.external_references_hashes"]
+    ext_refs = []
+
+    ext_refs_columns = (col for col in ext_refs_table.c if col.key != "id")
+    stmt = sa.select(*ext_refs_columns).where(ext_refs_table.c.id == stix_id)
+    ext_refs_results = conn.execute(stmt)
+    for ext_ref_mapping in ext_refs_results.mappings():
+        # make a dict; we will need to modify this mapping
+        ext_ref_dict = dict(ext_ref_mapping)
+        hash_ref_id = ext_ref_dict.pop("hash_ref_id")
+
+        hashes_dict = _read_hashes(hash_ref_id, ext_refs_hashes_table, conn)
+        if hashes_dict:
+            ext_ref_dict["hashes"] = hashes_dict
+
+        ext_refs.append(ext_ref_dict)
+
+    return ext_refs
+
+
+def _read_object_marking_refs(stix_id, stix_type_class, metadata, conn):
+    """
+    Read object marking refs from one of a couple special tables in the common
+    schema.
+
+    :param stix_id: A STIX ID, used to filter table rows
+    :param stix_type_class: STIXTypeClass enum value, used to determine whether
+        to read the table for SDOs or SCOs
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: The references as a list of strings
+    """
+
+    marking_table_name = "object_marking_refs_"
+    if stix_type_class is stix2.utils.STIXTypeClass.SCO:
+        marking_table_name += "sco"
+    else:
+        marking_table_name += "sdo"
+
+    # The SCO/SDO object_marking_refs tables are mostly identical; they just
+    # have different foreign key constraints (to different core tables).
+    marking_table = metadata.tables["common." + marking_table_name]
+
+    stmt = sa.select(marking_table.c.ref_id).where(marking_table.c.id == stix_id)
+    refs = conn.scalars(stmt).all()
+
+    return refs
+
+
+def _read_granular_markings(stix_id, stix_type_class, metadata, conn):
+    """
+    Read granular markings from one of a couple special tables in the common
+    schema.
+
+    :param stix_id: A STIX ID, used to filter table rows
+    :param stix_type_class: STIXTypeClass enum value, used to determine whether
+        to read the table for SDOs or SCOs
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: Granular markings as a list of dicts
+    """
+
+    marking_table_name = "granular_marking_"
+    if stix_type_class is stix2.utils.STIXTypeClass.SCO:
+        marking_table_name += "sco"
+    else:
+        marking_table_name += "sdo"
+
+    marking_table = metadata.tables["common." + marking_table_name]
+
+    stmt = sa.select(
+        marking_table.c.lang,
+        marking_table.c.marking_ref,
+        marking_table.c.selectors,
+    ).where(marking_table.c.id == stix_id)
+
+    marking_dicts = conn.execute(stmt).mappings().all()
+    return marking_dicts
+
+
+def _read_simple_array(fk_id, elt_column_name, array_table, conn):
+    """
+    Read array elements from a given table.
+
+    :param fk_id: A foreign key value used to find the correct array elements
+    :param elt_column_name: The name of the table column which contains the
+        array elements
+    :param array_table: A SQLAlchemy Table object containing the array data
+    :param conn: An SQLAlchemy DB connection
+    :return: The array, as a list
+    """
+    stmt = sa.select(array_table.c[elt_column_name]).where(array_table.c.id == fk_id)
+    refs = conn.scalars(stmt).all()
+    return refs
+
+
+def _read_kill_chain_phases(stix_id, type_table, metadata, conn):
+    """
+    Read kill chain phases from a table.
+
+    :param stix_id: A STIX ID used to filter table rows
+    :param type_table: A "parent" table whose name is used to compute the
+        kill chain phases table name
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: Kill chain phases as a list of dicts
+    """
+
+    kill_chain_phases_table = metadata.tables[type_table.fullname + "_kill_chain_phase"]
+    stmt = sa.select(
+        kill_chain_phases_table.c.kill_chain_name,
+        kill_chain_phases_table.c.phase_name
+    ).where(kill_chain_phases_table.c.id == stix_id)
+
+    kill_chain_phases = conn.execute(stmt).mappings().all()
+    return kill_chain_phases
+
+
+def _read_dictionary_property(stix_id, type_table, prop_name, prop_instance, metadata, conn):
+    """
+    Read a dictionary from a table.
+
+    :param stix_id: A STIX ID, used to filter table rows
+    :param type_table: A "parent" table whose name is used to compute the name
+        of the dictionary table
+    :param prop_name: The dictionary property name
+    :param prop_instance: The dictionary property instance
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: The dictionary, or None if no dictionary entries were found
+    """
+    dict_table_name = f"{type_table.fullname}_{prop_name}"
+    dict_table = metadata.tables[dict_table_name]
+
+    if len(prop_instance.valid_types) == 1:
+        stmt = sa.select(
+            dict_table.c.name, dict_table.c.value
+        ).where(
+            dict_table.c.id == stix_id
+        )
+
+        results = conn.execute(stmt)
+        dict_value = dict(results.all())
+
+    else:
+        # In this case, we get one column per valid type
+        type_cols = (col for col in dict_table.c if col.key not in ("id", "name"))
+        stmt = sa.select(dict_table.c.name, *type_cols).where(dict_table.c.id == stix_id)
+        results = conn.execute(stmt)
+
+        dict_value = {}
+        for row in results:
+            key, *type_values = row
+            # Exactly one of the type columns should be non-None; get that one
+            non_null_values = (v for v in type_values if v is not None)
+            first_non_null_value = next(non_null_values, None)
+            if first_non_null_value is None:
+                raise DataSourceError(
+                    f'In dictionary table {dict_table.fullname}, key "{key}"'
+                    " did not map to a non-null value"
+                )
+
+            dict_value[key] = first_non_null_value
+
+    # DictionaryProperty doesn't like empty dicts.
+    dict_value = dict_value or None
+
+    return dict_value
+
+
+def _read_embedded_object(obj_id, parent_table, embedded_type, metadata, conn):
+    """
+    Read an embedded object from the database.
+
+    :param obj_id: An ID value used to identify a particular embedded object,
+        used to filter table rows
+    :param parent_table: A "parent" table whose name is used to compute the
+        name of the embedded object table
+    :param embedded_type: The Python class used to represent the embedded
+        type (a _STIXBase subclass)
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: An instance of embedded_type
+    """
+
+    embedded_table_name = canonicalize_table_name(
+        f"{parent_table.name}_{embedded_type.__name__}",
+        parent_table.schema
+    )
+    embedded_table = metadata.tables[embedded_table_name]
+
+    # The PK column in this case is a bookkeeping column and does not
+    # correspond to an actual embedded object property.  So don't select
+    # that one.
+    non_id_cols = (col for col in embedded_table.c if col.key != "id")
+
+    stmt = sa.select(*non_id_cols).where(embedded_table.c.id == obj_id)
+    mapping_row = conn.execute(stmt).mappings().first()
+
+    if mapping_row is None:
+        obj = None
+
+    else:
+        obj_dict = dict(mapping_row)
+
+        for prop_name, prop_instance in embedded_type._properties.items():
+            if prop_name not in obj_dict:
+                prop_value = _read_complex_property_value(
+                    obj_id,
+                    prop_name,
+                    prop_instance,
+                    embedded_table,
+                    metadata,
+                    conn
+                )
+
+                if prop_value is not None:
+                    obj_dict[prop_name] = prop_value
+
+        obj = embedded_type(**obj_dict, allow_custom=True)
+
+    return obj
+
+
+def _read_embedded_object_list(fk_id, join_table, embedded_type, metadata, conn):
+    """
+    Read a list of embedded objects from database tables.
+
+    :param fk_id: A foreign key ID used to filter rows from the join table,
+        which acts to find relevant embedded objects
+    :param join_table: An SQLAlchemy Table object which is the required join
+        table
+    :param embedded_type: The Python class used to represent the list element
+        embedded type (a _STIXBase subclass)
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: A list of instances of embedded_type
+    """
+
+    embedded_table_name = canonicalize_table_name(
+        f"{join_table.name}_{embedded_type.__name__}",
+        join_table.schema
+    )
+    embedded_table = metadata.tables[embedded_table_name]
+
+    stmt = sa.select(embedded_table).join(join_table).where(join_table.c.id == fk_id)
+    results = conn.execute(stmt)
+    obj_list = []
+    for result_mapping in results.mappings():
+        obj_dict = dict(result_mapping)
+        obj_id = obj_dict.pop("id")
+
+        for prop_name, prop_instance in embedded_type._properties.items():
+            if prop_name not in obj_dict:
+                prop_value = _read_complex_property_value(
+                    obj_id,
+                    prop_name,
+                    prop_instance,
+                    embedded_table,
+                    metadata,
+                    conn
+                )
+
+                if prop_value is not None:
+                    obj_dict[prop_name] = prop_value
+
+        obj = embedded_type(**obj_dict, allow_custom=True)
+        obj_list.append(obj)
+
+    return obj_list
+
+
+def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, metadata, conn):
+    """
+    Read property values which require auxiliary tables to store.  These are
+    idiosyncratic and just require a lot of special cases.  This function has
+    no special support for top-level common properties, so it is more
+    general-purpose, suitable for any sort of object (top level or embedded).
+
+    :param obj_id: An ID of the owning object.  Would be a STIX ID for a
+        top-level object, but could also be something else for sub-objects.
+        Used as a foreign key value in queries, so we only get values for this
+        object.
+    :param prop_name: The name of the property to read
+    :param prop_instance: A Property (subclass) instance with property
+        config information
+    :param obj_table: The table for the owning object.  Mainly used for its
+        name; auxiliary table names are based on this
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: The property value
+    """
+
+    prop_value = None
+
+    if isinstance(prop_instance, stix2.properties.ListProperty):
+
+        if isinstance(prop_instance.contained, stix2.properties.ReferenceProperty):
+            ref_table_name = f"{obj_table.fullname}_{prop_name}"
+            ref_table = metadata.tables[ref_table_name]
+            prop_value = _read_simple_array(obj_id, "ref_id", ref_table, conn)
+
+        elif isinstance(prop_instance.contained, stix2.properties.EnumProperty):
+            enum_table_name = f"{obj_table.fullname}_{prop_name}"
+            enum_table = metadata.tables[enum_table_name]
+            prop_value = _read_simple_array(obj_id, prop_name, enum_table, conn)
+
+        elif isinstance(prop_instance.contained, stix2.properties.EmbeddedObjectProperty):
+            join_table_name = f"{obj_table.fullname}_{prop_name}"
+            join_table = metadata.tables[join_table_name]
+            prop_value = _read_embedded_object_list(
+                obj_id,
+                join_table,
+                prop_instance.contained.type,
+                metadata,
+                conn
+            )
+
+        elif inspect.isclass(prop_instance.contained) and issubclass(prop_instance.contained, stix2.KillChainPhase):
+            prop_value = _read_kill_chain_phases(obj_id, obj_table, metadata, conn)
+
+        else:
+            raise DataSourceError(
+                f'Not implemented: read "{prop_name}" property value'
+                f" of type list-of {prop_instance.contained}"
+            )
+
+    elif isinstance(prop_instance, stix2.properties.HashesProperty):
+        hashes_table_name = f"{obj_table.fullname}_{prop_name}"
+        hashes_table = metadata.tables[hashes_table_name]
+        prop_value = _read_hashes(obj_id, hashes_table, conn)
+
+    elif isinstance(prop_instance, stix2.properties.ExtensionsProperty):
+        # TODO: add support for extensions
+        pass
+
+    elif isinstance(prop_instance, stix2.properties.DictionaryProperty):
+        # ExtensionsProperty/HashesProperty subclasses DictionaryProperty, so
+        # this must come after those
+        prop_value = _read_dictionary_property(obj_id, obj_table, prop_name, prop_instance, metadata, conn)
+
+    elif isinstance(prop_instance, stix2.properties.EmbeddedObjectProperty):
+        prop_value = _read_embedded_object(
+            obj_id,
+            obj_table,
+            prop_instance.type,
+            metadata,
+            conn
+        )
+
+    else:
+        raise DataSourceError(
+            f'Not implemented: read "{prop_name}" property value'
+            f" of type {prop_instance.__class__}"
+        )
+
+    return prop_value
+
+
+def _read_complex_top_level_property_value(stix_id, stix_type_class, prop_name, prop_instance, type_table, metadata, conn):
+    """
+    Read property values which require auxiliary tables to store.  These
+    require a lot of special cases.  This function has additional support for
+    reading top-level common properties, which use special fixed tables.
+
+    :param stix_id: STIX ID of an object to read
+    :param stix_type_class: The kind of object (SCO, SDO, etc).  Which DB
+        tables to read can depend on this.
+    :param prop_name: The name of the property to read
+    :param prop_instance: A Property (subclass) instance with property
+        config information
+    :param type_table: The non-core base table used for this STIX type.  Mainly
+        used for its name; auxiliary table names are based on this
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: The property value
+    """
+
+    # Common properties: these use a fixed set of tables for all STIX objects
+    if prop_name == "external_references":
+        prop_value = _read_external_references(stix_id, metadata, conn)
+
+    elif prop_name == "object_marking_refs":
+        prop_value = _read_object_marking_refs(stix_id, stix_type_class, metadata, conn)
+
+    elif prop_name == "granular_markings":
+        prop_value = _read_granular_markings(stix_id, stix_type_class, metadata, conn)
+
+    else:
+        # Other properties use specific table patterns depending on property type
+        prop_value = _read_complex_property_value(stix_id, prop_name, prop_instance, type_table, metadata, conn)
+
+    return prop_value
+
+
+def read_object(stix_id, metadata, conn):
+    """
+    Read a STIX object from the database, identified by a STIX ID.
+
+    :param stix_id: A STIX ID
+    :param metadata: SQLAlchemy Metadata object containing all the table
+        information
+    :param conn: An SQLAlchemy DB connection
+    :return: A STIX object
+    """
+    _check_support(stix_id)
+
+    stix_class = _stix2_class_for(stix_id)
+
+    if not stix_class:
+        stix_type = stix2.utils.get_type_from_id(stix_id)
+        raise DataSourceError("Can't find registered class for type: " + stix_type)
+
+    core_table, type_table = _tables_for(stix_class, metadata)
+
+    if type_table.schema == "common":
+        # Applies to extension-definition SMO, whose data is stored in the
+        # common schema; it does not get its own.  This type class is used to
+        # determine which markings tables to use; its markings are
+        # in the *_sdo tables.
+        stix_type_class = stix2.utils.STIXTypeClass.SDO
+    else:
+        stix_type_class = stix2.utils.to_enum(type_table.schema, stix2.utils.STIXTypeClass)
+
+    simple_props = _read_simple_properties(stix_id, core_table, type_table, conn)
+    if simple_props is None:
+        # could not find anything for the given ID!
+        return None
+
+    obj_dict = dict(simple_props)
+    obj_dict["type"] = stix_class._type
+
+    for prop_name, prop_instance in stix_class._properties.items():
+        if prop_name not in obj_dict:
+            prop_value = _read_complex_top_level_property_value(
+                stix_id,
+                stix_type_class,
+                prop_name,
+                prop_instance,
+                type_table,
+                metadata,
+                conn
+            )
+
+            if prop_value is not None:
+                obj_dict[prop_name] = prop_value
+
+    stix_obj = stix_class(**obj_dict, allow_custom=True)
+    return stix_obj
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 434e37f8..282b7de7 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -9,6 +9,7 @@
 from stix2.datastore.relational_db.input_creation import (
     generate_insert_for_object,
 )
+from stix2.datastore.relational_db.query import read_object
 from stix2.datastore.relational_db.table_creation import create_table_objects
 from stix2.datastore.relational_db.utils import (
     canonicalize_table_name, schema_for, table_name_for,
@@ -80,7 +81,6 @@ def __init__(
                 them.
         """
         database_connection = create_engine(database_connection_url)
-        print(database_connection)
         self.metadata = MetaData()
         create_table_objects(
              self.metadata, stix_object_classes,
@@ -257,44 +257,14 @@ def __init__(
             )
 
     def get(self, stix_id, version=None, _composite_filters=None):
+        with self.database_connection.connect() as conn:
+            stix_obj = read_object(
+                stix_id,
+                self.metadata,
+                conn
+            )
 
-        stix_type = stix2.utils.get_type_from_id(stix_id)
-        stix_class = stix2.registry.class_for_type(
-            # TODO: give user control over STIX version used?
-            stix_type, stix_version=stix2.DEFAULT_VERSION,
-        )
-
-        # Info about the type-specific table
-        type_table_name = table_name_for(stix_type)
-        type_schema_name = schema_for(stix_class)
-        type_table = self.metadata.tables[f"{type_schema_name}.{type_table_name}"]
-
-        # Some fixed info about core tables
-        if type_schema_name == "sco":
-            core_table_name = "common.core_sco"
-        else:
-            # for SROs and SMOs too?
-            core_table_name = "common.core_sdo"
-
-        core_table = self.metadata.tables[core_table_name]
-
-        # Both core and type-specific tables have "id"; let's not duplicate
-        # that in the result set columns.  Is there a better way to do this?
-        type_cols_except_id = (
-            col for col in type_table.c if col.key != "id"
-        )
-
-        core_type_select = select(core_table, *type_cols_except_id) \
-            .join(type_table) \
-            .where(core_table.c.id == stix_id)
-
-        obj_dict = {}
-        with self.database_connection.begin() as conn:
-            # Should be at most one matching row
-            sco_data = conn.execute(core_type_select).mappings().first()
-            obj_dict.update(sco_data)
-
-        return stix_class(**obj_dict, allow_custom=True)
+        return stix_obj
 
     def all_versions(self, stix_id, version=None, _composite_filters=None):
         pass
diff --git a/stix2/properties.py b/stix2/properties.py
index f8e79ac9..59a5da1a 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -547,7 +547,9 @@ def clean(self, value, allow_custom=False, strict=False):
 class HexProperty(Property):
 
     def clean(self, value, allow_custom=False, strict=False):
-        if not re.match(r"^([a-fA-F0-9]{2})+$", value):
+        if isinstance(value, (bytes, bytearray)):
+            value = value.hex()
+        elif not re.match(r"^([a-fA-F0-9]{2})+$", value):
             raise ValueError("must contain an even number of hexadecimal characters")
         return value, False
 
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 39b461e6..f963229a 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -1,12 +1,21 @@
+import contextlib
+import datetime
 import json
 import os
 
+import pytest
+
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
+from stix2.datastore import DataSourceError
 import stix2.properties
+import stix2.registry
+import stix2.v21
+
+_DB_CONNECT_URL = f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
 
 store = RelationalDBStore(
-    f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
+    _DB_CONNECT_URL,
     True,
     None,
     False
@@ -51,8 +60,6 @@ def test_encrypted_artifact():
     read_obj = json.loads(store.get(artifact_stix_object['id']).serialize())
 
     for attrib in encrypted_artifact_dict.keys():
-        if attrib == 'hashes':  # TODO hashes are saved to separate table, functionality to retrieve is WIP
-            continue
         assert encrypted_artifact_dict[attrib] == read_obj[attrib]
 
 
@@ -98,8 +105,6 @@ def test_directory():
     read_obj = json.loads(store.get(directory_obj['id']).serialize())
 
     for attrib in directory_dict.keys():
-        if attrib == "contains_refs":  # TODO remove skip once we can pull from table join
-            continue
         if attrib == "ctime" or attrib == "mtime":  # convert both into stix2 date format for consistency
             assert stix2.utils.parse_into_datetime(directory_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
@@ -196,9 +201,9 @@ def test_email_addr():
     "cc_refs": ["email-addr--e4ee5301-b52d-59cd-a8fa-8036738c7194"],
     "subject": "Check out this picture of a cat!",
     "additional_header_fields": {
-        "Content-Disposition": "inline",
-        "X-Mailer": "Mutt/1.5.23",
-        "X-Originating-IP": "198.51.100.3",
+        "Content-Disposition": ["inline"],
+        "X-Mailer": ["Mutt/1.5.23"],
+        "X-Originating-IP": ["198.51.100.3"],
     },
     "body_multipart": [
         {
@@ -226,9 +231,6 @@ def test_email_msg():
     read_obj = json.loads(store.get(email_msg_stix_object['id']).serialize())
 
     for attrib in email_msg_dict.keys():
-        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-                or attrib == "additional_header_fields":  # join multiple tables not implemented yet
-            continue
         if attrib == "date":
             assert stix2.utils.parse_into_datetime(email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
                 read_obj[attrib],
@@ -243,9 +245,6 @@ def test_multipart_email_msg():
     read_obj = json.loads(store.get(multipart_email_msg_stix_object['id']).serialize())
 
     for attrib in multipart_email_msg_dict.keys():
-        if attrib == "to_refs" or attrib == "cc_refs" or attrib == "bcc_refs" \
-                or attrib == "additional_header_fields" or attrib == "body_multipart":  # join multiple tables not implemented yet
-            continue
         if attrib == "date":
             assert stix2.utils.parse_into_datetime(multipart_email_msg_dict[attrib]) == stix2.utils.parse_into_datetime(
                 read_obj[attrib],
@@ -267,6 +266,7 @@ def test_multipart_email_msg():
     "name": "qwerty.dll",
     "size": 25536,
     "name_enc": "windows-1252",
+    "magic_number_hex": "a1b2c3",
     "mime_type": "application/msword",
     "ctime": "2018-11-23T08:17:27.000Z",
     "mtime": "2018-11-23T08:17:27.000Z",
@@ -284,8 +284,6 @@ def test_file():
     read_obj = json.loads(store.get(file_stix_object['id']).serialize())
 
     for attrib in file_dict.keys():
-        if attrib == "contains_refs" or attrib == "hashes":  # join multiple tables not implemented yet
-            continue
         if attrib == "ctime" or attrib == "mtime" or attrib == "atime":
             assert stix2.utils.parse_into_datetime(file_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
@@ -381,8 +379,6 @@ def test_network_traffic():
     read_obj = store.get(network_traffic_stix_object['id'])
 
     for attrib in network_traffic_dict.keys():
-        if attrib == "encapsulates_refs":  # multiple table join not implemented
-            continue
         if attrib == "start" or attrib == "end":
             assert stix2.utils.parse_into_datetime(network_traffic_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
@@ -421,9 +417,6 @@ def test_process():
     read_obj = json.loads(store.get(process_stix_object['id']).serialize())
 
     for attrib in process_dict.keys():
-        if attrib == "child_refs" or attrib == "opened_connection_refs" or attrib == "environment_variables":
-            # join multiple tables not implemented yet
-            continue
         if attrib == "created_time":
             assert stix2.utils.parse_into_datetime(process_dict[attrib]) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
@@ -536,8 +529,6 @@ def test_windows_registry():
     read_obj = json.loads(store.get(windows_registry_stix_object['id']).serialize())
 
     for attrib in windows_registry_dict.keys():
-        if attrib == "values":  # skip multiple table join
-            continue
         if attrib == "modified_time":
             assert stix2.utils.parse_into_datetime(windows_registry_dict[attrib]) == stix2.utils.parse_into_datetime(
                 read_obj[attrib],
@@ -610,11 +601,360 @@ def test_x509_certificate_with_extensions():
     read_obj = json.loads(store.get(extensions_x509_certificate_stix_object['id']).serialize())
 
     for attrib in extensions_x509_certificate_dict.keys():
-        if attrib == "x509_v3_extensions":  # skipping multi-table join
-            continue
         if attrib == "validity_not_before" or attrib == "validity_not_after":
             assert stix2.utils.parse_into_datetime(
                 extensions_x509_certificate_dict[attrib],
             ) == stix2.utils.parse_into_datetime(read_obj[attrib])
             continue
         assert extensions_x509_certificate_dict[attrib] == read_obj[attrib]
+
+
+def test_source_get_not_exists():
+    obj = store.get("identity--00000000-0000-0000-0000-000000000000")
+    assert obj is None
+
+
+def test_source_no_registration():
+    with pytest.raises(DataSourceError):
+        # error, since no registered class can be found
+        store.get("doesnt-exist--a9e52398-3312-4377-90c2-86d49446c0d0")
+
+
+def _unregister(reg_section, stix_type, ext_id=None):
+    """
+    Unregister a class from the stix2 library's registry.
+
+    :param reg_section: A registry section; depends on the kind of
+        class which was registered
+    :param stix_type: A STIX type
+    :param ext_id: An extension-definition ID, if applicable.  A second
+        unregistration will occur in the extensions section of the registry if
+        given.
+    """
+    # We ought to have a library function for this...
+    del stix2.registry.STIX2_OBJ_MAPS["2.1"][reg_section][stix_type]
+    if ext_id:
+        del stix2.registry.STIX2_OBJ_MAPS["2.1"]["extensions"][ext_id]
+
+
+@contextlib.contextmanager
+def _register_object(*args, **kwargs):
+    """
+    A contextmanager which can register a class for an SDO/SRO and ensure it is
+    unregistered afterword.
+
+    :param args: Positional args to a @CustomObject decorator
+    :param kwargs: Keyword args to a @CustomObject decorator
+    :return: The registered class
+    """
+    @stix2.CustomObject(*args, **kwargs)
+    class TestClass:
+        pass
+
+    try:
+        yield TestClass
+    except:
+        ext_id = kwargs.get("extension_name")
+        if not ext_id and len(args) >= 3:
+            ext_id = args[2]
+
+        _unregister("objects", TestClass._type, ext_id)
+
+        raise
+
+
+@contextlib.contextmanager
+def _register_observable(*args, **kwargs):
+    """
+    A contextmanager which can register a class for an SCO and ensure it is
+    unregistered afterword.
+
+    :param args: Positional args to a @CustomObservable decorator
+    :param kwargs: Keyword args to a @CustomObservable decorator
+    :return: The registered class
+    """
+    @stix2.CustomObservable(*args, **kwargs)
+    class TestClass:
+        pass
+
+    try:
+        yield TestClass
+    except:
+        ext_id = kwargs.get("extension_name")
+        if not ext_id and len(args) >= 4:
+            ext_id = args[3]
+
+        _unregister("observables", TestClass._type, ext_id)
+
+        raise
+
+
+# "Base" properties used to derive property variations for testing (e.g. in a
+# list, in a dictionary, in an embedded object, etc).  Also includes sample
+# values used to create test objects.  The keys here are used to parameterize a
+# fixture below.  Parameterizing fixtures via simple strings makes for more
+# understandable unit test output, although it can be kind of awkward in the
+# implementation (can require long if-then chains checking the parameter
+# strings).
+_TEST_PROPERTIES = {
+    "binary": (stix2.properties.BinaryProperty(), "Af9J"),
+    "boolean": (stix2.properties.BooleanProperty(), True),
+    "float": (stix2.properties.FloatProperty(), 1.23),
+    "hex": (stix2.properties.HexProperty(), "a1b2c3"),
+    "integer": (stix2.properties.IntegerProperty(), 1),
+    "string": (stix2.properties.StringProperty(), "test"),
+    "timestamp": (
+        stix2.properties.TimestampProperty(),
+        datetime.datetime.now(tz=datetime.timezone.utc)
+    ),
+    "ref": (
+        stix2.properties.ReferenceProperty("SDO"),
+        "identity--ec83b570-0743-4179-a5e3-66fd2fae4711"
+    ),
+    "enum": (
+        stix2.properties.EnumProperty(["value1", "value2"]),
+        "value1"
+    )
+}
+
+
+@pytest.fixture(params=_TEST_PROPERTIES.keys())
+def base_property_value(request):
+    """Produce basic property instances and test values."""
+
+    base = _TEST_PROPERTIES.get(request.param)
+    if not base:
+        pytest.fail("Unrecognized base property: " + request.param)
+
+    return base
+
+
+@pytest.fixture(
+    params=[
+        "base",
+        "list-of",
+        "dict-of",
+        # The following two test nesting lists inside dicts and vice versa
+        "dict-list-of",
+        "list-dict-of",
+        "subobject",
+        "list-of-subobject-prop",
+        "list-of-subobject-class"
+    ]
+)
+def property_variation_value(request, base_property_value):
+    """
+    Produce property variations (and corresponding value variations) based on a
+    base property instance and value.  E.g. in a list, in a sub-object, etc.
+    """
+    base_property, prop_value = base_property_value
+
+    class Embedded(stix2.v21._STIXBase21):
+        """
+        Used for property variations where the property is embedded in a
+        sub-object.
+        """
+        _properties = {
+            "embedded": base_property
+        }
+
+    if request.param == "base":
+        prop_variation = base_property
+        prop_variation_value = prop_value
+
+    elif request.param == "list-of":
+        prop_variation = stix2.properties.ListProperty(base_property)
+        prop_variation_value = [prop_value]
+
+    elif request.param == "dict-of":
+        prop_variation = stix2.properties.DictionaryProperty(
+            # DictionaryProperty.valid_types does not accept property
+            # instances (except ListProperty instances), only classes...
+            valid_types=type(base_property)
+        )
+        # key name doesn't matter here
+        prop_variation_value = {"key": prop_value}
+
+    elif request.param == "dict-list-of":
+        prop_variation = stix2.properties.DictionaryProperty(
+            valid_types=stix2.properties.ListProperty(base_property)
+        )
+        # key name doesn't matter here
+        prop_variation_value = {"key": [prop_value]}
+
+    elif request.param == "list-dict-of":
+        # These seem to all fail... perhaps there is no intent to support
+        # this?
+        pytest.xfail("ListProperty(DictionaryProperty) not supported?")
+
+        # prop_variation = stix2.properties.ListProperty(
+        #     stix2.properties.DictionaryProperty(valid_types=type(base_property))
+        # )
+        # key name doesn't matter here
+        # prop_variation_value = [{"key": prop_value}]
+
+    elif request.param == "subobject":
+        prop_variation = stix2.properties.EmbeddedObjectProperty(Embedded)
+        prop_variation_value = {"embedded": prop_value}
+
+    elif request.param == "list-of-subobject-prop":
+        # list-of-embedded values via EmbeddedObjectProperty
+        prop_variation = stix2.properties.ListProperty(
+            stix2.properties.EmbeddedObjectProperty(Embedded)
+        )
+        prop_variation_value = [{"embedded": prop_value}]
+
+    elif request.param == "list-of-subobject-class":
+        # Skip all of these since we know the data sink currently chokes on it
+        pytest.xfail("Data sink doesn't yet support ListProperty(<_STIXBase subclass>)")
+
+        # list-of-embedded values using the embedded class directly
+        # prop_variation = stix2.properties.ListProperty(Embedded)
+        # prop_variation_value = [{"embedded": prop_value}]
+
+    else:
+        pytest.fail("Unrecognized property variation: " + request.param)
+
+    return prop_variation, prop_variation_value
+
+
+@pytest.fixture(params=["sdo", "sco", "sro"])
+def object_variation(request, property_variation_value):
+    """
+    Create and register a custom class variation (SDO, SCO, etc), then
+    instantiate it and produce the resulting object.
+    """
+
+    property_instance, property_value = property_variation_value
+
+    # Fixed extension ID for everything
+    ext_id = "extension-definition--15de9cdb-3515-4271-8479-8141154c5647"
+
+    if request.param == "sdo":
+        @stix2.CustomObject(
+            "test-object", [
+                ("prop_name", property_instance)
+            ],
+            ext_id,
+            is_sdo=True
+        )
+        class TestClass:
+            pass
+
+    elif request.param == "sro":
+        @stix2.CustomObject(
+            "test-object", [
+                ("prop_name", property_instance)
+            ],
+            ext_id,
+            is_sdo=False
+        )
+        class TestClass:
+            pass
+
+    elif request.param == "sco":
+        @stix2.CustomObservable(
+            "test-object", [
+                ("prop_name", property_instance)
+            ],
+            ["prop_name"],
+            ext_id
+        )
+        class TestClass:
+            pass
+
+    else:
+        pytest.fail("Unrecognized object variation: " + request.param)
+
+    try:
+        instance = TestClass(prop_name=property_value)
+        yield instance
+    finally:
+        reg_section = "observables" if request.param == "sco" else "objects"
+        _unregister(reg_section, TestClass._type, ext_id)
+
+
+def test_property(object_variation):
+    """
+    Try to more exhaustively test many different property configurations:
+    ensure schemas can be created and values can be stored and retrieved.
+    """
+    rdb_store = RelationalDBStore(
+        _DB_CONNECT_URL,
+        True,
+        None,
+        True,
+        True,
+        type(object_variation)
+    )
+
+    rdb_store.add(object_variation)
+    read_obj = rdb_store.get(object_variation["id"])
+
+    assert read_obj == object_variation
+
+
+def test_dictionary_property_complex():
+    """
+    Test a dictionary property with multiple valid_types
+    """
+    with _register_object(
+        "test-object", [
+            ("prop_name",
+                 stix2.properties.DictionaryProperty(
+                     valid_types=[
+                         stix2.properties.IntegerProperty,
+                         stix2.properties.FloatProperty,
+                         stix2.properties.StringProperty
+                     ]
+                 )
+             )
+        ],
+        "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
+        is_sdo=True
+    ) as cls:
+
+        obj = cls(
+            prop_name={"a": 1, "b": 2.3, "c": "foo"}
+        )
+
+        rdb_store = RelationalDBStore(
+            _DB_CONNECT_URL,
+            True,
+            None,
+            True,
+            True,
+            cls
+        )
+
+        rdb_store.add(obj)
+        read_obj = rdb_store.get(obj["id"])
+        assert read_obj == obj
+
+
+def test_extension_definition():
+    obj = stix2.ExtensionDefinition(
+        created_by_ref="identity--8a5fb7e4-aabe-4635-8972-cbcde1fa4792",
+        name="test",
+        schema="a schema",
+        version="1.2.3",
+        extension_types=["property-extension", "new-sdo", "new-sro"],
+        object_marking_refs=[
+            "marking-definition--caa0d913-5db8-4424-aae0-43e770287d30",
+            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77"
+        ],
+        granular_markings=[
+            {
+                "lang": "en_US",
+                "selectors": ["name", "schema"]
+            },
+            {
+                "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
+                "selectors": ["name", "schema"]
+            }
+        ]
+    )
+
+    store.add(obj)
+    read_obj = store.get(obj["id"])
+    assert read_obj == obj

From c33d879a43e7e239c9d641ceb0d621c00dd612c7 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Fri, 13 Sep 2024 12:44:52 -0400
Subject: [PATCH 114/132] Remove a stray comma

---
 stix2/test/v21/test_datastore_relational_db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index f963229a..78dd9f3d 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -12,7 +12,7 @@
 import stix2.registry
 import stix2.v21
 
-_DB_CONNECT_URL = f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres",
+_DB_CONNECT_URL = f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres"
 
 store = RelationalDBStore(
     _DB_CONNECT_URL,

From 16aaf9cad5de007fcc7ac9d255c58081f2f55a33 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Fri, 13 Sep 2024 15:14:17 -0400
Subject: [PATCH 115/132] add id binding for dictionaries

---
 stix2/datastore/relational_db/input_creation.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 006b4fed..e469a7f2 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -38,10 +38,7 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
     schema_name = kwargs.get("schema_name")
     foreign_key_value = kwargs.get("foreign_key_value")
     insert_statements = list()
-    if "id" in stix_object:
-        bindings["id"] = stix_object["id"]
-    elif foreign_key_value:
-        bindings["id"] = foreign_key_value
+
     table = data_sink.tables_dictionary[
         canonicalize_table_name(
             table_name + "_" + dictionary_name,
@@ -54,6 +51,10 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
     valid_types = stix_object._properties[dictionary_name].valid_types
     for name, value in stix_object[dictionary_name].items():
         bindings = dict()
+        if "id" in stix_object:
+            bindings["id"] = stix_object["id"]
+        elif foreign_key_value:
+            bindings["id"] = foreign_key_value
         if not valid_types or len(self.valid_types) == 1:
             value_binding = "value"
         elif isinstance(value, int) and IntegerProperty in valid_types:

From f0f6d8fdd1925944d2ef0b6c6cda8f52a7b67b61 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Sun, 15 Sep 2024 17:25:26 -0400
Subject: [PATCH 116/132] Fix DictionaryProperty so that EnumProperty and
 ReferenceProperty both work as valid_types, and fix some bugs.  -
 DictionaryProperty.__init__()'s valid_types argument now accepts either   
 property classes or instances  - DictionaryProperty now correctly propagates
 allow_custom down to its    valid_types properties, and propagates their
 returned has_custom flag    upward  - DictionaryProperty's cleaned
 dictionaries actually contain the cleaned    values now (before, the cleaned
 values were ignored)

Property class cleanup:
 - Property subclass .clean() methods now are correct overrides of the
   base class method, including parameter naming and default values
 - OpenVocabProperty now doesn't treat any value as custom, regardless of
   the allow_custom setting, which was our original intent.  A violation
   of strict mode now causes ValueError to be raised, not
   CustomContentError.
 - BooleanProperty.clean() now honors strict=True (it will only accept
   values True and False in that case).

Added various property tests of strictness, and additional
DictionaryProperty tests which exercise custom value detection when used
with ReferenceProperty.
---
 stix2/properties.py               | 224 +++++++++++++++++++-----------
 stix2/test/test_properties.py     |  72 +++++++++-
 stix2/test/v21/test_properties.py | 173 +++++++++++++++--------
 3 files changed, 322 insertions(+), 147 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 59a5da1a..424912ec 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -172,7 +172,7 @@ class Property(object):
 
     """
 
-    def _default_clean(self, value, allow_custom=False):
+    def _default_clean(self, value, allow_custom=False, strict=False):
         if value != self._fixed_value:
             raise ValueError("must equal '{}'.".format(self._fixed_value))
         return value, False
@@ -226,7 +226,7 @@ def __init__(self, contained, **kwargs):
 
         super(ListProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom, strict_flag=False):
+    def clean(self, value, allow_custom=False, strict=False):
         try:
             iter(value)
         except TypeError:
@@ -240,7 +240,7 @@ def clean(self, value, allow_custom, strict_flag=False):
         if isinstance(self.contained, Property):
             for item in value:
                 try:
-                    valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict_flag)
+                    valid, temp_custom = self.contained.clean(item, allow_custom, strict=strict)
                 except TypeError:
                     valid, temp_custom = self.contained.clean(item, allow_custom)
                 result.append(valid)
@@ -281,11 +281,10 @@ def __init__(self, **kwargs):
         super(StringProperty, self).__init__(**kwargs)
 
     def clean(self, value, allow_custom=False, strict=False):
-        if not isinstance(value, str):
-            if strict is True:
-                raise ValueError("Must be a string.")
+        if strict and not isinstance(value, str):
+            raise ValueError("Must be a string.")
 
-            value = str(value)
+        value = str(value)
         return value, False
 
 
@@ -320,7 +319,7 @@ def __init__(self, min=None, max=None, **kwargs):
         super(IntegerProperty, self).__init__(**kwargs)
 
     def clean(self, value, allow_custom=False, strict=False):
-        if strict is True and not isinstance(value, int):
+        if strict and not isinstance(value, int):
             raise ValueError("must be an integer.")
 
         try:
@@ -347,7 +346,7 @@ def __init__(self, min=None, max=None, **kwargs):
         super(FloatProperty, self).__init__(**kwargs)
 
     def clean(self, value, allow_custom=False, strict=False):
-        if strict is True and not isinstance(value, float):
+        if strict and not isinstance(value, float):
             raise ValueError("must be a float.")
 
         try:
@@ -372,6 +371,9 @@ class BooleanProperty(Property):
 
     def clean(self, value, allow_custom=False, strict=False):
 
+        if strict and not isinstance(value, bool):
+            raise ValueError("must be a boolean value.")
+
         if isinstance(value, str):
             value = value.lower()
 
@@ -403,74 +405,134 @@ class DictionaryProperty(Property):
 
     def __init__(self, valid_types=None, spec_version=DEFAULT_VERSION, **kwargs):
         self.spec_version = spec_version
+        self.valid_types = self._normalize_valid_types(valid_types or [])
 
-        simple_types = [
-            BinaryProperty, BooleanProperty, FloatProperty, HexProperty, IntegerProperty, StringProperty,
-            TimestampProperty, ReferenceProperty, EnumProperty,
-        ]
-        if not valid_types:
-            valid_types = [Property]
-        else:
-            if not isinstance(valid_types, list):
-                valid_types = [valid_types]
-            for type_ in valid_types:
-                if isinstance(type_, ListProperty):
-                    found = False
-                    for simple_type in simple_types:
-                        if isinstance(type_.contained, simple_type):
-                            found = True
-                    if not found:
-                        raise ValueError("Dictionary Property does not support lists of type: ", type_.contained, type(type_.contained))
-                elif type_ not in simple_types:
-                    raise ValueError("Dictionary Property does not support this value's type: ", type_)
+        super(DictionaryProperty, self).__init__(**kwargs)
 
-        self.valid_types = valid_types
+    def _normalize_valid_types(self, valid_types):
+        """
+        Normalize valid_types to a list of property instances.  Also ensure any
+        property types given are supported for type enforcement.
 
-        super(DictionaryProperty, self).__init__(**kwargs)
+        :param valid_types: A single or iterable of Property instances or
+            subclasses
+        :return: A list of Property instances, or None if none were given
+        """
+        simple_types = (
+            BinaryProperty, BooleanProperty, FloatProperty, HexProperty,
+            IntegerProperty, StringProperty, TimestampProperty,
+            ReferenceProperty, EnumProperty,
+        )
 
-    def clean(self, value, allow_custom=False):
+        # Normalize single prop instances/classes to lists
+        try:
+            iter(valid_types)
+        except TypeError:
+            valid_types = [valid_types]
+
+        prop_instances = []
+        for valid_type in valid_types:
+            if inspect.isclass(valid_type):
+                # Note: this will fail as of this writing with EnumProperty
+                # ReferenceProperty, ListProperty.  Callers must instantiate
+                # those with suitable settings themselves.
+                prop_instance = valid_type()
+
+            else:
+                prop_instance = valid_type
+
+            # ListProperty's element type must be one of the supported
+            # simple types.
+            if isinstance(prop_instance, ListProperty):
+                if not isinstance(prop_instance.contained, simple_types):
+                    raise ValueError(
+                        "DictionaryProperty does not support lists of type: "
+                        + type(prop_instance.contained).__name__
+                    )
+
+            elif not isinstance(prop_instance, simple_types):
+                raise ValueError(
+                    "DictionaryProperty does not support value type: "
+                    + type(prop_instance).__name__
+                )
+
+            prop_instances.append(prop_instance)
+
+        return prop_instances or None
+
+    def _check_dict_key(self, k):
+        if self.spec_version == '2.0':
+            if len(k) < 3:
+                raise DictionaryKeyError(k, "shorter than 3 characters")
+            elif len(k) > 256:
+                raise DictionaryKeyError(k, "longer than 256 characters")
+        elif self.spec_version == '2.1':
+            if len(k) > 250:
+                raise DictionaryKeyError(k, "longer than 250 characters")
+        if not re.match(r"^[a-zA-Z0-9_-]+$", k):
+            msg = (
+                "contains characters other than lowercase a-z, "
+                "uppercase A-Z, numerals 0-9, hyphen (-), or "
+                "underscore (_)"
+            )
+            raise DictionaryKeyError(k, msg)
+
+    def clean(self, value, allow_custom=False, strict=False):
         try:
             dictified = _get_dict(value)
         except ValueError:
             raise ValueError("The dictionary property must contain a dictionary")
 
-        for k in dictified.keys():
-            if self.spec_version == '2.0':
-                if len(k) < 3:
-                    raise DictionaryKeyError(k, "shorter than 3 characters")
-                elif len(k) > 256:
-                    raise DictionaryKeyError(k, "longer than 256 characters")
-            elif self.spec_version == '2.1':
-                if len(k) > 250:
-                    raise DictionaryKeyError(k, "longer than 250 characters")
-            if not re.match(r"^[a-zA-Z0-9_-]+$", k):
-                msg = (
-                    "contains characters other than lowercase a-z, "
-                    "uppercase A-Z, numerals 0-9, hyphen (-), or "
-                    "underscore (_)"
-                )
-                raise DictionaryKeyError(k, msg)
+        has_custom = False
+        for k, v in dictified.items():
 
-            clean = False
-            for type_ in self.valid_types:
-                if isinstance(type_, ListProperty):
-                    type_.clean(value=dictified[k], allow_custom=False, strict_flag=True)
-                    clean = True
-                else:
-                    type_instance = type_()
+            self._check_dict_key(k)
+
+            if self.valid_types:
+                for type_ in self.valid_types:
                     try:
-                        type_instance.clean(value=dictified[k], allow_custom=False, strict=True)
-                        clean = True
+                        # ReferenceProperty at least, does check for
+                        # customizations, so we must propagate that
+                        dictified[k], temp_custom = type_.clean(
+                            value=v,
+                            allow_custom=allow_custom,
+                            # Ignore the passed-in value and fix this to True;
+                            # we need strict cleaning to disambiguate value
+                            # types here.
+                            strict=True
+                        )
+                    except CustomContentError:
+                        # Need to propagate these, not treat as a type error
+                        raise
+                    except Exception as e:
+                        # clean failed; value must not conform to type_
+                        # Should be a narrower exception type here, but I don't
+                        # know if it's safe to assume any particular exception
+                        # types...
+                        pass
+                    else:
+                        # clean succeeded; should check the has_custom flag
+                        # just in case.  But if allow_custom is False, I expect
+                        # one of the valid_types property instances would have
+                        # already raised an exception.
+                        has_custom = has_custom or temp_custom
+                        if has_custom and not allow_custom:
+                            raise CustomContentError(f'Custom content detected in key "{k}"')
+
                         break
-                    except ValueError:
-                        continue
-            if not clean:
-                raise ValueError("Dictionary Property does not support this value's type: ", type(dictified[k]))
+
+                else:
+                    # clean failed for all properties!
+                    raise ValueError(
+                        f"Invalid value: {v!r}"
+                    )
+
+            # else: no valid types given, so we skip the validity check
 
         if len(dictified) < 1:
             raise ValueError("must not be empty.")
 
-        return dictified, False
+        return dictified, has_custom
 
 
 class HashesProperty(DictionaryProperty):
@@ -488,7 +550,7 @@ def __init__(self, spec_hash_names, spec_version=DEFAULT_VERSION, **kwargs):
             if alg:
                 self.__alg_to_spec_name[alg] = spec_hash_name
 
-    def clean(self, value, allow_custom, strict=False):
+    def clean(self, value, allow_custom=False, strict=False):
         # ignore the has_custom return value here; there is no customization
         # of DictionaryProperties.
         clean_dict, _ = super().clean(value, allow_custom)
@@ -597,7 +659,7 @@ def __init__(self, valid_types=None, invalid_types=None, spec_version=DEFAULT_VE
 
         super(ReferenceProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom=False, strict=False):
         if isinstance(value, _STIXBase):
             value = value.id
         value = str(value)
@@ -675,7 +737,7 @@ def clean(self, value, allow_custom):
 
 class SelectorProperty(Property):
 
-    def clean(self, value, allow_custom=False):
+    def clean(self, value, allow_custom=False, strict=False):
         if not SELECTOR_REGEX.match(value):
             raise ValueError("must adhere to selector syntax.")
         return value, False
@@ -696,7 +758,7 @@ def __init__(self, type, **kwargs):
         self.type = type
         super(EmbeddedObjectProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom=False, strict=False):
         if isinstance(value, dict):
             value = self.type(allow_custom=allow_custom, **value)
         elif not isinstance(value, self.type):
@@ -724,7 +786,7 @@ def __init__(self, allowed, **kwargs):
         self.allowed = allowed
         super(EnumProperty, self).__init__(**kwargs)
 
-    def clean(self, value, allow_custom, strict=False):
+    def clean(self, value, allow_custom=False, strict=False):
 
         cleaned_value, _ = super(EnumProperty, self).clean(value, allow_custom, strict)
 
@@ -746,26 +808,20 @@ def __init__(self, allowed, **kwargs):
             allowed = [allowed]
         self.allowed = allowed
 
-    def clean(self, value, allow_custom, strict=False):
+    def clean(self, value, allow_custom=False, strict=False):
         cleaned_value, _ = super(OpenVocabProperty, self).clean(
             value, allow_custom, strict,
         )
 
-        # Disabled: it was decided that enforcing this is too strict (might
-        # break too much user code).  Revisit when we have the capability for
-        # more granular config settings when creating objects.
-        #
-        if strict is True:
-            has_custom = cleaned_value not in self.allowed
-
-            if not allow_custom and has_custom:
-                raise CustomContentError(
-                    "custom value in open vocab: '{}'".format(cleaned_value),
-                )
+        # Customization enforcement is disabled: it was decided that enforcing
+        # it is too strict (might break too much user code).  On the other
+        # hand, we need to lock it down in strict mode.  If we are locking it
+        # down in strict mode, we always throw an exception if a value isn't
+        # in the vocab list, and never report anything as "custom".
+        if strict and cleaned_value not in self.allowed:
+            raise ValueError("not in vocab: " + cleaned_value)
 
-        has_custom = False
-
-        return cleaned_value, has_custom
+        return cleaned_value, False
 
 
 class PatternProperty(StringProperty):
@@ -780,7 +836,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
         self.spec_version = spec_version
         super(ObservableProperty, self).__init__(*args, **kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom=False, strict=False):
         try:
             dictified = _get_dict(value)
             # get deep copy since we are going modify the dict and might
@@ -828,7 +884,7 @@ class ExtensionsProperty(DictionaryProperty):
     def __init__(self, spec_version=DEFAULT_VERSION, required=False):
         super(ExtensionsProperty, self).__init__(spec_version=spec_version, required=required)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom=False, strict=False):
         try:
             dictified = _get_dict(value)
             # get deep copy since we are going modify the dict and might
@@ -894,7 +950,7 @@ def __init__(self, spec_version=DEFAULT_VERSION, *args, **kwargs):
         self.spec_version = spec_version
         super(STIXObjectProperty, self).__init__(*args, **kwargs)
 
-    def clean(self, value, allow_custom):
+    def clean(self, value, allow_custom=False, strict=False):
         # Any STIX Object (SDO, SRO, or Marking Definition) can be added to
         # a bundle with no further checks.
         stix2_classes = {'_DomainObject', '_RelationshipObject', 'MarkingDefinition'}
diff --git a/stix2/test/test_properties.py b/stix2/test/test_properties.py
index 5116a685..4637e39f 100644
--- a/stix2/test/test_properties.py
+++ b/stix2/test/test_properties.py
@@ -181,6 +181,12 @@ def test_string_property():
     assert prop.clean(1)
     assert prop.clean([1, 2, 3])
 
+    with pytest.raises(ValueError):
+        prop.clean(1, strict=True)
+
+    result = prop.clean("foo", strict=True)
+    assert result == ("foo", False)
+
 
 def test_type_property():
     prop = TypeProperty('my-type')
@@ -244,6 +250,15 @@ def test_integer_property_invalid(value):
         int_prop.clean(value)
 
 
+def test_integer_property_strict():
+    int_prop = IntegerProperty()
+    with pytest.raises(ValueError):
+        int_prop.clean("123", strict=True)
+
+    result = int_prop.clean(123, strict=True)
+    assert result == (123, False)
+
+
 @pytest.mark.parametrize(
     "value", [
         2,
@@ -253,8 +268,8 @@ def test_integer_property_invalid(value):
     ],
 )
 def test_float_property_valid(value):
-    int_prop = FloatProperty()
-    assert int_prop.clean(value) is not None
+    float_prop = FloatProperty()
+    assert float_prop.clean(value) is not None
 
 
 @pytest.mark.parametrize(
@@ -264,9 +279,18 @@ def test_float_property_valid(value):
     ],
 )
 def test_float_property_invalid(value):
-    int_prop = FloatProperty()
+    float_prop = FloatProperty()
     with pytest.raises(ValueError):
-        int_prop.clean(value)
+        float_prop.clean(value)
+
+
+def test_float_property_strict():
+    float_prop = FloatProperty()
+    with pytest.raises(ValueError):
+        float_prop.clean("1.323", strict=True)
+
+    result = float_prop.clean(1.323, strict=True)
+    assert result == (1.323, False)
 
 
 @pytest.mark.parametrize(
@@ -308,6 +332,15 @@ def test_boolean_property_invalid(value):
         bool_prop.clean(value)
 
 
+def test_boolean_property_strict():
+    bool_prop = BooleanProperty()
+    with pytest.raises(ValueError):
+        bool_prop.clean("true", strict=True)
+
+    result = bool_prop.clean(True, strict=True)
+    assert result == (True, False)
+
+
 @pytest.mark.parametrize(
     "value", [
         '2017-01-01T12:34:56Z',
@@ -368,6 +401,16 @@ def test_enum_property_invalid():
         enum_prop.clean('z', True)
 
 
+def test_enum_property_strict():
+    enum_prop = EnumProperty(['1', '2', '3'])
+    with pytest.raises(ValueError):
+        enum_prop.clean(1, strict=True)
+
+    result = enum_prop.clean(1, strict=False)
+    assert result == ("1", False)
+
+
+
 @pytest.mark.xfail(
     reason="Temporarily disabled custom open vocab enforcement",
     strict=True,
@@ -391,6 +434,27 @@ def test_openvocab_property(vocab):
     assert ov_prop.clean("d", True) == ("d", True)
 
 
+def test_openvocab_property_strict():
+    ov_prop = OpenVocabProperty(["1", "2", "3"])
+    with pytest.raises(ValueError):
+        ov_prop.clean(1, allow_custom=False, strict=True)
+
+    with pytest.raises(ValueError):
+        ov_prop.clean(1, allow_custom=True, strict=True)
+
+    result = ov_prop.clean("1", allow_custom=False, strict=True)
+    assert result == ("1", False)
+
+    result = ov_prop.clean(1, allow_custom=True, strict=False)
+    assert result == ("1", False)
+
+    result = ov_prop.clean(1, allow_custom=False, strict=False)
+    assert result == ("1", False)
+
+    result = ov_prop.clean("foo", allow_custom=False, strict=False)
+    assert result == ("foo", False)
+
+
 @pytest.mark.parametrize(
     "value", [
         {"sha256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b"},
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 0eff3c29..2cd42a12 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -6,71 +6,16 @@
     ExtraPropertiesError, ParseError,
 )
 from stix2.properties import (
-    DictionaryProperty, EmbeddedObjectProperty, ExtensionsProperty,
-    HashesProperty, IDProperty, IntegerProperty, ListProperty,
-    ObservableProperty, ReferenceProperty, STIXObjectProperty, StringProperty,
+    DictionaryProperty, EmbeddedObjectProperty, EnumProperty,
+    ExtensionsProperty, HashesProperty, IDProperty, IntegerProperty,
+    ListProperty, ObservableProperty, ReferenceProperty, STIXObjectProperty,
+    StringProperty,
 )
 from stix2.v21.common import MarkingProperty
 
 from . import constants
 
 
-def test_dictionary_property():
-    p = DictionaryProperty()
-
-    assert p.clean({'spec_version': '2.1'})
-    with pytest.raises(ValueError):
-        p.clean({}, False)
-
-
-def test_dictionary_property_values_str():
-    p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
-    result = p.clean({'x': '123'}, False)
-    assert result == ({'x': '123'}, False)
-
-    q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
-    with pytest.raises(ValueError):
-        assert q.clean({'x': [123]}, False)
-
-
-def test_dictionary_property_values_int():
-    p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
-    result = p.clean({'x': 123}, False)
-    assert result == ({'x': 123}, False)
-
-    q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
-    with pytest.raises(ValueError):
-        assert q.clean({'x': [123]}, False)
-
-
-def test_dictionary_property_values_stringlist():
-    p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
-    result = p.clean({'x': ['abc', 'def']}, False)
-    assert result == ({'x': ['abc', 'def']}, False)
-
-    q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
-    with pytest.raises(ValueError):
-        assert q.clean({'x': [123]})
-
-    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
-    with pytest.raises(ValueError):
-        assert r.clean({'x': [123, 456]})
-
-
-def test_dictionary_property_values_list():
-    p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
-    result = p.clean({'x': 123}, False)
-    assert result == ({'x': 123}, False)
-
-    q = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
-    result = q.clean({'x': '123'}, False)
-    assert result == ({'x': '123'}, False)
-
-    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
-    with pytest.raises(ValueError):
-        assert r.clean({'x': ['abc', 'def']}, False)
-
-
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'
 
@@ -436,6 +381,116 @@ class NewObj():
     assert test_obj.property1[0]['foo'] == 'bar'
 
 
+def test_dictionary_property():
+    p = DictionaryProperty()
+
+    result = p.clean({'spec_version': '2.1'})
+    assert result == ({'spec_version': '2.1'}, False)
+
+    with pytest.raises(ValueError):
+        p.clean({}, False)
+
+
+def test_dictionary_property_values_str():
+    p = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
+    result = p.clean({'x': '123'}, False)
+    assert result == ({'x': '123'}, False)
+
+    q = DictionaryProperty(valid_types=[StringProperty], spec_version='2.1')
+    with pytest.raises(ValueError):
+        assert q.clean({'x': [123]}, False)
+
+
+def test_dictionary_property_values_str_single():
+    # singles should be treated as length-one lists
+    p = DictionaryProperty(valid_types=StringProperty, spec_version='2.1')
+    result = p.clean({'x': '123'}, False)
+    assert result == ({'x': '123'}, False)
+
+    with pytest.raises(ValueError):
+        assert p.clean({'x': [123]}, False)
+
+
+def test_dictionary_property_values_int():
+    p = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
+    result = p.clean({'x': 123}, False)
+    assert result == ({'x': 123}, False)
+
+    q = DictionaryProperty(valid_types=[IntegerProperty], spec_version='2.1')
+    with pytest.raises(ValueError):
+        assert q.clean({'x': [123]}, False)
+
+
+def test_dictionary_property_values_stringlist():
+    p = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
+    result = p.clean({'x': ['abc', 'def']}, False)
+    assert result == ({'x': ['abc', 'def']}, False)
+
+    q = DictionaryProperty(valid_types=[ListProperty(StringProperty)], spec_version='2.1')
+    with pytest.raises(ValueError):
+        assert q.clean({'x': [123]})
+
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    with pytest.raises(ValueError):
+        assert r.clean({'x': [123, 456]})
+
+
+def test_dictionary_property_values_list():
+    p = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    result = p.clean({'x': 123}, False)
+    assert result == ({'x': 123}, False)
+
+    q = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    result = q.clean({'x': '123'}, False)
+    assert result == ({'x': '123'}, False)
+
+    r = DictionaryProperty(valid_types=[StringProperty, IntegerProperty], spec_version='2.1')
+    with pytest.raises(ValueError):
+        assert r.clean({'x': ['abc', 'def']}, False)
+
+
+def test_dictionary_property_ref_custom():
+    p = DictionaryProperty(
+        valid_types=ReferenceProperty(valid_types="SDO"), spec_version="2.1"
+    )
+
+    result = p.clean({"key": "identity--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=False)
+    assert result == ({"key": "identity--a2ac7670-f88f-424a-b3be-28f612f943f9"}, False)
+
+    with pytest.raises(ValueError):
+        p.clean({"key": "software--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=False)
+
+    with pytest.raises(ValueError):
+        p.clean({"key": "software--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=True)
+
+    pfoo = DictionaryProperty(
+        valid_types=ReferenceProperty(valid_types=["SDO", "foo"]), spec_version="2.1"
+    )
+
+    with pytest.raises(CustomContentError):
+        pfoo.clean({"key": "foo--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=False)
+
+    result = pfoo.clean({"key": "foo--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=True)
+    assert result == ({"key": "foo--a2ac7670-f88f-424a-b3be-28f612f943f9"}, True)
+
+
+def test_dictionary_property_values_strict_clean():
+    prop = DictionaryProperty(
+        valid_types=[EnumProperty(["value1", "value2"]), IntegerProperty]
+    )
+
+    result = prop.clean({"key": "value1"}, allow_custom=False)
+    assert result == ({"key": "value1"}, False)
+
+    result = prop.clean({"key": 123}, allow_custom=False)
+    assert result == ({"key": 123}, False)
+
+    with pytest.raises(ValueError):
+        # IntegerProperty normally cleans "123" to 123, but can't when used
+        # in a DictionaryProperty.
+        prop.clean({"key": "123"}, allow_custom=False)
+
+
 @pytest.mark.parametrize(
     "key", [
         "a",

From f29472721bf19dacfdc854716ada7a21dfe0d3e5 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 16 Sep 2024 13:02:47 -0400
Subject: [PATCH 117/132] handle dictionaries better

---
 .../datastore/relational_db/table_creation.py | 66 +++++++++++--------
 stix2/datastore/relational_db/utils.py        | 46 ++++++++-----
 2 files changed, 68 insertions(+), 44 deletions(-)

diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 79e61397..fc3e6b5d 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -343,35 +343,44 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
             nullable=False,
         ),
     )
-    if len(self.valid_types) == 1:
-        if not isinstance(self.valid_types[0], ListProperty):
-            columns.append(
-                Column(
-                    "value",
-                    # its a class
-                    determine_sql_type_from_class(self.valid_types[0]),
-                    nullable=False,
-                ),
-            )
+    if self.valid_types:
+        if len(self.valid_types) == 1:
+            if not isinstance(self.valid_types[0], ListProperty):
+                columns.append(
+                    Column(
+                        "value",
+                        # its a class
+                        determine_sql_type_from_class(self.valid_types[0]),
+                        nullable=False,
+                    ),
+                )
+            else:
+                contained_class = self.valid_types[0].contained
+                columns.append(
+                    Column(
+                        "value",
+                        # its an instance, not a class
+                        ARRAY(contained_class.determine_sql_type()),
+                        nullable=False,
+                    ),
+                )
         else:
-            contained_class = self.valid_types[0].contained
-            columns.append(
-                Column(
-                    "value",
-                    # its an instance, not a class
-                    ARRAY(contained_class.determine_sql_type()),
-                    nullable=False,
-                ),
-            )
+            for column_type in self.valid_types:
+                sql_type = determine_sql_type_from_class(column_type)
+                columns.append(
+                    Column(
+                        determine_column_name(column_type),
+                        sql_type,
+                    ),
+                )
     else:
-        for column_type in self.valid_types:
-            sql_type = determine_sql_type_from_class(column_type)
-            columns.append(
-                Column(
-                    determine_column_name(column_type),
-                    sql_type,
-                ),
-            )
+        columns.append(
+            Column(
+                "value",
+                Text,
+                nullable=False,
+            ),
+        )
     return [
         Table(
             canonicalize_table_name(table_name + "_" + name),
@@ -383,6 +392,9 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
     ]
 
 
+
+
+
 @add_method(EmbeddedObjectProperty)
 def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
     level = kwargs.get("level")
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 0ae0bf4e..6f2fc778 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -52,10 +52,20 @@ def canonicalize_table_name(table_name, schema_name=None):
     return inflection.underscore(full_name)
 
 
+_IGNORE_OBJECTS = [ "language-content"]
+
+
 def get_all_subclasses(cls):
     all_subclasses = []
 
     for subclass in cls.__subclasses__():
+        # This code might be useful if we decide that some objects just cannot have there tables
+        # automatically generated
+        
+        # if hasattr(subclass, "_type") and subclass._type in _IGNORE_OBJECTS:
+        #     print(f'It is currently not possible to create a table for {subclass._type}')
+        #     return []
+        # else:
         all_subclasses.append(subclass)
         all_subclasses.extend(get_all_subclasses(subclass))
     return all_subclasses
@@ -122,37 +132,39 @@ def flat_classes(class_or_classes):
         yield class_or_classes
 
 
-def determine_sql_type_from_class(cls):  # noqa: F811
-    if cls == BinaryProperty:
+def determine_sql_type_from_class(cls_or_inst):  # noqa: F811
+    if cls_or_inst == BinaryProperty or isinstance(cls_or_inst, BinaryProperty):
         return LargeBinary
-    elif cls == BooleanProperty:
+    elif cls_or_inst == BooleanProperty or isinstance(cls_or_inst, BooleanProperty):
         return Boolean
-    elif cls == FloatProperty:
+    elif cls_or_inst == FloatProperty or isinstance(cls_or_inst, FloatProperty):
         return Float
-    elif cls == HexProperty:
+    elif cls_or_inst == HexProperty or isinstance(cls_or_inst, HexProperty):
         return LargeBinary
-    elif cls == IntegerProperty:
+    elif cls_or_inst == IntegerProperty or isinstance(cls_or_inst, IntegerProperty):
         return Integer
-    elif cls == StringProperty or cls == ReferenceProperty:
+    elif (cls_or_inst == StringProperty or cls_or_inst == ReferenceProperty or
+          isinstance(cls_or_inst, StringProperty)  or isinstance(cls_or_inst, ReferenceProperty)):
         return Text
-    elif cls == TimestampProperty:
+    elif cls_or_inst == TimestampProperty or isinstance(cls_or_inst, TimestampProperty):
         return TIMESTAMP(timezone=True)
-    elif cls == Property:
+    elif cls_or_inst == Property or isinstance(cls_or_inst, Property):
         return Text
 
 
-def determine_column_name(cls):  # noqa: F811
-    if cls == BinaryProperty:
+def determine_column_name(cls_or_inst):  # noqa: F811
+    if cls_or_inst == BinaryProperty or isinstance(cls_or_inst, BinaryProperty):
         return "binary_value"
-    elif cls == BooleanProperty:
+    elif cls_or_inst == BooleanProperty or isinstance(cls_or_inst, BooleanProperty):
         return "boolean_value"
-    elif cls == FloatProperty:
+    elif cls_or_inst == FloatProperty or isinstance(cls_or_inst, FloatProperty):
         return "float_value"
-    elif cls == HexProperty:
+    elif cls_or_inst == HexProperty or isinstance(cls_or_inst, HexProperty):
         return "hex_value"
-    elif cls == IntegerProperty:
+    elif cls_or_inst == IntegerProperty or isinstance(cls_or_inst, IntegerProperty):
         return "integer_value"
-    elif cls == StringProperty or cls == ReferenceProperty:
+    elif (cls_or_inst == StringProperty or cls_or_inst == ReferenceProperty or
+          isinstance(cls_or_inst, StringProperty)  or isinstance(cls_or_inst, ReferenceProperty)):
         return "string_value"
-    elif cls == TimestampProperty:
+    elif cls_or_inst == TimestampProperty or isinstance(cls_or_inst, TimestampProperty):
         return "timestamp_value"

From c847a5c282463b35484cd48c9ad0c18eb207f923 Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Mon, 16 Sep 2024 17:51:46 -0400
Subject: [PATCH 118/132] Minor fix to certain relational data sink unit tests
 related to dictionary properties, to account for fixes to relational data
 sink. Also fix a minor typo in DictionaryProperty where an exception variable
 was left in by mistake...

---
 stix2/properties.py                            | 2 +-
 stix2/test/v21/test_datastore_relational_db.py | 4 +---
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/stix2/properties.py b/stix2/properties.py
index 424912ec..eef757be 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -504,7 +504,7 @@ def clean(self, value, allow_custom=False, strict=False):
                     except CustomContentError:
                         # Need to propagate these, not treat as a type error
                         raise
-                    except Exception as e:
+                    except Exception:
                         # clean failed; value must not conform to type_
                         # Should be a narrower exception type here, but I don't
                         # know if it's safe to assume any particular exception
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 78dd9f3d..0c2e3607 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -768,9 +768,7 @@ class Embedded(stix2.v21._STIXBase21):
 
     elif request.param == "dict-of":
         prop_variation = stix2.properties.DictionaryProperty(
-            # DictionaryProperty.valid_types does not accept property
-            # instances (except ListProperty instances), only classes...
-            valid_types=type(base_property)
+            valid_types=base_property
         )
         # key name doesn't matter here
         prop_variation_value = {"key": prop_value}

From e508f99875a486c1685639359ee7c0a4ac439fa7 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 17 Sep 2024 10:53:46 -0400
Subject: [PATCH 119/132] fix insert of dictionary values

---
 .../datastore/relational_db/input_creation.py | 20 +++++++---
 stix2/datastore/relational_db/utils.py        | 38 ++++++++++---------
 2 files changed, 35 insertions(+), 23 deletions(-)

diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index e469a7f2..aa7e98bf 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -30,6 +30,16 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
     return {name: stix_object[name]}
 
 
+
+def instance_in_valid_types(cls, valid_types):
+    for v in valid_types:
+        if isinstance(v, cls):
+            return True
+    return False
+
+def is_valid_type(cls, valid_types):
+    return cls in valid_types or instance_in_valid_types(cls, valid_types)
+
 @add_method(DictionaryProperty)
 def generate_insert_information(self, dictionary_name, stix_object, **kwargs):  # noqa: F811
     bindings = dict()
@@ -57,15 +67,15 @@ def generate_insert_information(self, dictionary_name, stix_object, **kwargs):
             bindings["id"] = foreign_key_value
         if not valid_types or len(self.valid_types) == 1:
             value_binding = "value"
-        elif isinstance(value, int) and IntegerProperty in valid_types:
+        elif isinstance(value, int) and is_valid_type(IntegerProperty, valid_types):
             value_binding = "integer_value"
-        elif isinstance(value, str) and StringProperty in valid_types:
+        elif isinstance(value, str) and is_valid_type(StringProperty, valid_types):
             value_binding = "string_value"
-        elif isinstance(value, bool) and BooleanProperty in valid_types:
+        elif isinstance(value, bool) and is_valid_type(BooleanProperty, valid_types):
             value_binding = "boolean_value"
-        elif isinstance(value, float) and FloatProperty in valid_types:
+        elif isinstance(value, float) and is_valid_type(FloatProperty, valid_types):
             value_binding = "float_value"
-        elif isinstance(value, STIXdatetime) and TimestampProperty in valid_types:
+        elif isinstance(value, STIXdatetime) and is_valid_type(TimestampProperty, valid_types):
             value_binding = "timestamp_value"
         else:
             value_binding = "string_value"
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 6f2fc778..5d06de7b 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -61,7 +61,7 @@ def get_all_subclasses(cls):
     for subclass in cls.__subclasses__():
         # This code might be useful if we decide that some objects just cannot have there tables
         # automatically generated
-        
+
         # if hasattr(subclass, "_type") and subclass._type in _IGNORE_OBJECTS:
         #     print(f'It is currently not possible to create a table for {subclass._type}')
         #     return []
@@ -132,39 +132,41 @@ def flat_classes(class_or_classes):
         yield class_or_classes
 
 
+def is_class_or_instance(cls_or_inst, cls):
+    return cls_or_inst == cls or isinstance(cls_or_inst, cls)
+
+
 def determine_sql_type_from_class(cls_or_inst):  # noqa: F811
-    if cls_or_inst == BinaryProperty or isinstance(cls_or_inst, BinaryProperty):
+    if is_class_or_instance(cls_or_inst, BinaryProperty):
         return LargeBinary
-    elif cls_or_inst == BooleanProperty or isinstance(cls_or_inst, BooleanProperty):
+    elif is_class_or_instance(cls_or_inst, BooleanProperty):
         return Boolean
-    elif cls_or_inst == FloatProperty or isinstance(cls_or_inst, FloatProperty):
+    elif is_class_or_instance(cls_or_inst, FloatProperty ):
         return Float
-    elif cls_or_inst == HexProperty or isinstance(cls_or_inst, HexProperty):
+    elif is_class_or_instance(cls_or_inst, HexProperty):
         return LargeBinary
-    elif cls_or_inst == IntegerProperty or isinstance(cls_or_inst, IntegerProperty):
+    elif is_class_or_instance(cls_or_inst, IntegerProperty):
         return Integer
-    elif (cls_or_inst == StringProperty or cls_or_inst == ReferenceProperty or
-          isinstance(cls_or_inst, StringProperty)  or isinstance(cls_or_inst, ReferenceProperty)):
+    elif is_class_or_instance(cls_or_inst, StringProperty) or is_class_or_instance(cls_or_inst, ReferenceProperty):
         return Text
-    elif cls_or_inst == TimestampProperty or isinstance(cls_or_inst, TimestampProperty):
+    elif is_class_or_instance(cls_or_inst, TimestampProperty):
         return TIMESTAMP(timezone=True)
-    elif cls_or_inst == Property or isinstance(cls_or_inst, Property):
+    elif is_class_or_instance(cls_or_inst, Property):
         return Text
 
 
 def determine_column_name(cls_or_inst):  # noqa: F811
-    if cls_or_inst == BinaryProperty or isinstance(cls_or_inst, BinaryProperty):
+    if is_class_or_instance(cls_or_inst, BinaryProperty):
         return "binary_value"
-    elif cls_or_inst == BooleanProperty or isinstance(cls_or_inst, BooleanProperty):
+    elif is_class_or_instance(cls_or_inst, BooleanProperty):
         return "boolean_value"
-    elif cls_or_inst == FloatProperty or isinstance(cls_or_inst, FloatProperty):
+    elif is_class_or_instance(cls_or_inst, FloatProperty):
         return "float_value"
-    elif cls_or_inst == HexProperty or isinstance(cls_or_inst, HexProperty):
+    elif is_class_or_instance(cls_or_inst, HexProperty):
         return "hex_value"
-    elif cls_or_inst == IntegerProperty or isinstance(cls_or_inst, IntegerProperty):
+    elif is_class_or_instance(cls_or_inst, IntegerProperty):
         return "integer_value"
-    elif (cls_or_inst == StringProperty or cls_or_inst == ReferenceProperty or
-          isinstance(cls_or_inst, StringProperty)  or isinstance(cls_or_inst, ReferenceProperty)):
+    elif is_class_or_instance(cls_or_inst, StringProperty) or is_class_or_instance(cls_or_inst, ReferenceProperty):
         return "string_value"
-    elif cls_or_inst == TimestampProperty or isinstance(cls_or_inst, TimestampProperty):
+    elif is_class_or_instance(cls_or_inst, TimestampProperty):
         return "timestamp_value"

From f1af0dd778ee91b0892aab113cbc2e3a800b13f6 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Tue, 5 Nov 2024 17:09:48 -0500
Subject: [PATCH 120/132] initial db backend

---
 stix2/datastore/__init__.py                   |  2 +-
 .../database_backend_base.py                  | 35 +++++++
 .../database_backends/postgres_backend.py     | 38 +++++++
 .../datastore/relational_db/relational_db.py  | 98 +++++++++----------
 .../relational_db/relational_db_testing.py    |  7 +-
 5 files changed, 126 insertions(+), 54 deletions(-)
 create mode 100644 stix2/datastore/relational_db/database_backends/database_backend_base.py
 create mode 100644 stix2/datastore/relational_db/database_backends/postgres_backend.py

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 715c6e6b..39229805 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -212,7 +212,7 @@ def add(self, *args, **kwargs):
         """
         try:
             return self.sink.add(*args, **kwargs)
-        except AttributeError:
+        except AttributeError as ex:
             msg = "%s has no data sink to put objects in"
             raise AttributeError(msg % self.__class__.__name__)
 
diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
new file mode 100644
index 00000000..f4e791e8
--- /dev/null
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -0,0 +1,35 @@
+
+from typing import Any
+import os
+
+from sqlalchemy import create_engine
+from sqlalchemy_utils import create_database, database_exists, drop_database
+
+
+class DatabaseBackend:
+    def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
+        self.database_connection_url = database_connection_url
+        self.database_exists = database_exists(database_connection_url)
+
+        if force_recreate:
+            if self.database_exists:
+                drop_database(database_connection_url)
+            create_database(database_connection_url)
+            self.database_exists = database_exists(database_connection_url)
+
+        self.database_connection = create_engine(database_connection_url)
+
+    def _create_schemas(self):
+        pass
+
+    @staticmethod
+    def _determine_schema_name(stix_object):
+        return ""
+
+    def _create_database(self):
+        if self.database_exists:
+            drop_database(self.database_connection.url)
+        create_database(self.database_connection.url)
+        self.database_exists = database_exists(self.database_connection.url)
+
+
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
new file mode 100644
index 00000000..b9672f97
--- /dev/null
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -0,0 +1,38 @@
+import os
+from typing import Any
+from sqlalchemy.schema import CreateSchema
+
+from .database_backend_base import DatabaseBackend
+
+from stix2.base import (
+    _DomainObject, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
+)
+
+
+class PostgresBackend(DatabaseBackend):
+    default_database_connection_url = \
+        f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:" + \
+        f"{os.getenv('POSTGRES_PASSWORD', 'postgres')}@" + \
+        f"{os.getenv('POSTGRES_IP_ADDRESS', '0.0.0.0')}:" + \
+        f"{os.getenv('POSTGRES_PORT', '5432')}/postgres"
+
+    def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any):
+        super().__init__(database_connection_url, force_recreate=False, **kwargs)
+
+    def _create_schemas(self):
+        with self.database_connection.begin() as trans:
+            trans.execute(CreateSchema("common", if_not_exists=True))
+            trans.execute(CreateSchema("sdo", if_not_exists=True))
+            trans.execute(CreateSchema("sco", if_not_exists=True))
+            trans.execute(CreateSchema("sro", if_not_exists=True))
+
+    @staticmethod
+    def _determine_schema_name(stix_object):
+        if isinstance(stix_object, _DomainObject):
+            return "sdo"
+        elif isinstance(stix_object, _Observable):
+            return "sco"
+        elif isinstance(stix_object, _RelationshipObject):
+            return "sro"
+        elif isinstance(stix_object, _MetaObject):
+            return "common"
\ No newline at end of file
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 282b7de7..3e09d33b 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -58,8 +58,8 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 
 class RelationalDBStore(DataStoreMixin):
     def __init__(
-        self, database_connection_url, allow_custom=True, version=None,
-        instantiate_database=True, force_recreate=False, *stix_object_classes,
+        self, db_backend, allow_custom=True, version=None,
+        instantiate_database=True, *stix_object_classes,
     ):
         """
         Initialize this store.
@@ -80,7 +80,7 @@ def __init__(
                 auto-detect all classes and create table schemas for all of
                 them.
         """
-        database_connection = create_engine(database_connection_url)
+
         self.metadata = MetaData()
         create_table_objects(
              self.metadata, stix_object_classes,
@@ -88,15 +88,14 @@ def __init__(
 
         super().__init__(
             source=RelationalDBSource(
-                database_connection,
+                db_backend,
                 metadata=self.metadata,
             ),
             sink=RelationalDBSink(
-                database_connection,
+                db_backend,
                 allow_custom=allow_custom,
                 version=version,
                 instantiate_database=instantiate_database,
-                force_recreate=force_recreate,
                 metadata=self.metadata,
             ),
         )
@@ -104,8 +103,8 @@ def __init__(
 
 class RelationalDBSink(DataSink):
     def __init__(
-        self, database_connection_or_url, allow_custom=True, version=None,
-        instantiate_database=True, force_recreate=False, *stix_object_classes, metadata=None,
+        self, db_backend, allow_custom=True, version=None,
+        instantiate_database=True, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this sink.  Only one of stix_object_classes and metadata
@@ -135,14 +134,16 @@ def __init__(
         """
         super(RelationalDBSink, self).__init__()
 
-        if isinstance(database_connection_or_url, str):
-            self.database_connection = create_engine(database_connection_or_url)
-        else:
-            self.database_connection = database_connection_or_url
+        self.db_backend = db_backend
 
-        self.database_exists = database_exists(self.database_connection.url)
-        if force_recreate:
-            self._create_database()
+        # if isinstance(database_connection_or_url, str):
+        #     self.database_connection = create_engine(database_connection_or_url)
+        # else:
+        #     self.database_connection = database_connection_or_url
+
+        # self.database_exists = database_exists(self.database_connection.url)
+        # if force_recreate:
+        #     self._create_database()
 
         if metadata:
             self.metadata = metadata
@@ -160,49 +161,49 @@ def __init__(
             self.tables_dictionary[canonicalize_table_name(t.name, t.schema)] = t
 
         if instantiate_database:
-            if not self.database_exists:
-                self._create_database()
-            self._create_schemas()
+            if not self.db_backend.database_exists:
+                self.db_backend._create_database()
+            self.db_backend._create_schemas()
             self._instantiate_database()
 
-    def _create_schemas(self):
-        with self.database_connection.begin() as trans:
-            trans.execute(CreateSchema("common", if_not_exists=True))
-            trans.execute(CreateSchema("sdo", if_not_exists=True))
-            trans.execute(CreateSchema("sco", if_not_exists=True))
-            trans.execute(CreateSchema("sro", if_not_exists=True))
+    # def _create_schemas(self):
+    #     with self.database_connection.begin() as trans:
+    #         trans.execute(CreateSchema("common", if_not_exists=True))
+    #         trans.execute(CreateSchema("sdo", if_not_exists=True))
+    #         trans.execute(CreateSchema("sco", if_not_exists=True))
+    #         trans.execute(CreateSchema("sro", if_not_exists=True))
 
     def _instantiate_database(self):
-        self.metadata.create_all(self.database_connection)
+        self.metadata.create_all(self.db_backend.database_connection)
 
-    def _create_database(self):
-        if self.database_exists:
-            drop_database(self.database_connection.url)
-        create_database(self.database_connection.url)
-        self.database_exists = database_exists(self.database_connection.url)
+    # def _create_database(self):
+    #     if self.database_exists:
+    #         drop_database(self.database_connection.url)
+    #     create_database(self.database_connection.url)
+    #     self.database_exists = database_exists(self.database_connection.url)
 
     def generate_stix_schema(self):
         for t in self.metadata.tables.values():
-            print(CreateTable(t).compile(self.database_connection))
+            print(CreateTable(t).compile(self.db_backend.database_connection))
 
     def add(self, stix_data, version=None):
         _add(self, stix_data)
     add.__doc__ = _add.__doc__
 
-    @staticmethod
-    def _determine_schema_name(stix_object):
-        if isinstance(stix_object, _DomainObject):
-            return "sdo"
-        elif isinstance(stix_object, _Observable):
-            return "sco"
-        elif isinstance(stix_object, _RelationshipObject):
-            return "sro"
-        elif isinstance(stix_object, _MetaObject):
-            return "common"
+    # @staticmethod
+    # def _determine_schema_name(stix_object):
+    #     if isinstance(stix_object, _DomainObject):
+    #         return "sdo"
+    #     elif isinstance(stix_object, _Observable):
+    #         return "sco"
+    #     elif isinstance(stix_object, _RelationshipObject):
+    #         return "sro"
+    #     elif isinstance(stix_object, _MetaObject):
+    #         return "common"
 
     def insert_object(self, stix_object):
-        schema_name = self._determine_schema_name(stix_object)
-        with self.database_connection.begin() as trans:
+        schema_name = self.db_backend._determine_schema_name(stix_object)
+        with self.db_backend.database_connection.begin() as trans:
             statements = generate_insert_for_object(self, stix_object, schema_name)
             for stmt in statements:
                 print("executing: ", stmt)
@@ -211,7 +212,7 @@ def insert_object(self, stix_object):
 
     def clear_tables(self):
         tables = list(reversed(self.metadata.sorted_tables))
-        with self.database_connection.begin() as trans:
+        with self.db_backend.database_connection.begin() as trans:
             for table in tables:
                 delete_stmt = delete(table)
                 print(f'delete_stmt: {delete_stmt}')
@@ -220,7 +221,7 @@ def clear_tables(self):
 
 class RelationalDBSource(DataSource):
     def __init__(
-        self, database_connection_or_url, *stix_object_classes, metadata=None,
+        self, db_backend, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this source.  Only one of stix_object_classes and metadata
@@ -243,10 +244,7 @@ def __init__(
         """
         super().__init__()
 
-        if isinstance(database_connection_or_url, str):
-            self.database_connection = create_engine(database_connection_or_url)
-        else:
-            self.database_connection = database_connection_or_url
+        self.db_backend = db_backend
 
         if metadata:
             self.metadata = metadata
@@ -257,7 +255,7 @@ def __init__(
             )
 
     def get(self, stix_id, version=None, _composite_filters=None):
-        with self.database_connection.connect() as conn:
+        with self.db_backend.connect() as conn:
             stix_obj = read_object(
                 stix_id,
                 self.metadata,
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 3e720ace..aadb3a0b 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -6,6 +6,8 @@
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
+from database_backends.postgres_backend import PostgresBackend
+
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
     path_enc="latin1",
@@ -251,14 +253,13 @@ def test_dictionary():
 
 def main():
     store = RelationalDBStore(
-        "postgresql://localhost/stix-data-sink",
+        PostgresBackend("postgresql://localhost/stix-data-sink"),
         False,
         None,
         True,
-        True,
     )
 
-    if store.sink.database_exists:
+    if store.sink.db_backend.database_exists:
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 

From 8fd7226f27f04d61190b366676a237b31385475f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 7 Nov 2024 13:14:52 -0500
Subject: [PATCH 121/132] handle ARRAY

---
 .../database_backend_base.py                  |  64 ++-
 .../database_backends/postgres_backend.py     |  30 +-
 .../datastore/relational_db/relational_db.py  |   6 +-
 .../relational_db/relational_db_testing.py    | 120 +++---
 .../datastore/relational_db/table_creation.py | 386 ++++++++++--------
 stix2/datastore/relational_db/utils.py        |  24 +-
 6 files changed, 374 insertions(+), 256 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index f4e791e8..5b65119b 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -4,7 +4,10 @@
 
 from sqlalchemy import create_engine
 from sqlalchemy_utils import create_database, database_exists, drop_database
-
+from sqlalchemy import (  # create_engine,; insert,
+    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
+    Integer, LargeBinary, Table, Text, UniqueConstraint,
+)
 
 class DatabaseBackend:
     def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
@@ -20,7 +23,7 @@ def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any)
         self.database_connection = create_engine(database_connection_url)
 
     def _create_schemas(self):
-        pass
+        return
 
     @staticmethod
     def _determine_schema_name(stix_object):
@@ -32,4 +35,61 @@ def _create_database(self):
         create_database(self.database_connection.url)
         self.database_exists = database_exists(self.database_connection.url)
 
+    def schema_for(self, stix_class):
+        return ""
+
+    @staticmethod
+    def determine_sql_type_for_property():  # noqa: F811
+        pass
+
+    @staticmethod
+    def determine_sql_type_for_kill_chain_phase():  # noqa: F811
+        return None
+
+    @staticmethod
+    def determine_sql_type_for_binary_property():  # noqa: F811
+        return Text
+
+    @staticmethod
+    def determine_sql_type_for_boolean_property():  # noqa: F811
+        return Boolean
+
+    @staticmethod
+    def determine_sql_type_for_float_property():  # noqa: F811
+        return Float
+
+    @staticmethod
+    def determine_sql_type_for_hex_property():  # noqa: F811
+        return LargeBinary
+
+    @staticmethod
+    def determine_sql_type_for_integer_property():  # noqa: F811
+        return Integer
+
+    @staticmethod
+    def determine_sql_type_for_reference_property():  # noqa: F811
+        return Text
+
+    @staticmethod
+    def determine_sql_type_for_string_property():  # noqa: F811
+        return Text
+
+    @staticmethod
+    def determine_sql_type_for_timestamp_property():  # noqa: F811
+        return TIMESTAMP(timezone=True)
+
+    @staticmethod
+    def determine_sql_type_for_key_as_int():  # noqa: F811
+        return Integer
+
+
+    @staticmethod
+    def determine_sql_type_for_key_as_id():  # noqa: F811
+        return Text
+
+    @staticmethod
+    def array_allowed():
+        return False
+
+
 
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index b9672f97..08ca3c9e 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -1,13 +1,26 @@
 import os
 from typing import Any
 from sqlalchemy.schema import CreateSchema
+from sqlalchemy import (  # create_engine,; insert,
+    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
+    Integer, LargeBinary, Table, Text, UniqueConstraint,
+)
 
-from .database_backend_base import DatabaseBackend
+from stix2.datastore.relational_db.utils import schema_for
 
 from stix2.base import (
-    _DomainObject, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
+)
+
+from stix2.properties import (
+    BinaryProperty, BooleanProperty, DictionaryProperty,
+    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
+    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
+    ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
+    TimestampProperty, TypeProperty,
 )
 
+from .database_backend_base import DatabaseBackend
 
 class PostgresBackend(DatabaseBackend):
     default_database_connection_url = \
@@ -35,4 +48,15 @@ def _determine_schema_name(stix_object):
         elif isinstance(stix_object, _RelationshipObject):
             return "sro"
         elif isinstance(stix_object, _MetaObject):
-            return "common"
\ No newline at end of file
+            return "common"
+
+    def schema_for(self, stix_class):
+        return schema_for(stix_class)
+
+
+    @staticmethod
+    def array_allowed():
+        return True
+
+
+
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 3e09d33b..6fbb027d 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -83,7 +83,7 @@ def __init__(
 
         self.metadata = MetaData()
         create_table_objects(
-             self.metadata, stix_object_classes,
+             self.metadata, db_backend, stix_object_classes,
         )
 
         super().__init__(
@@ -251,11 +251,11 @@ def __init__(
         else:
             self.metadata = MetaData()
             create_table_objects(
-                self.metadata, stix_object_classes,
+                self.metadata, db_backend, stix_object_classes,
             )
 
     def get(self, stix_id, version=None, _composite_filters=None):
-        with self.db_backend.connect() as conn:
+        with self.db_backend.database_connection.connect() as conn:
             stix_obj = read_object(
                 stix_id,
                 self.metadata,
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index aadb3a0b..fe35a454 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -194,61 +194,61 @@ def custom_obj():
     return obj
 
 
-@stix2.CustomObject(
-    "test-object", [
-        ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
-    ],
-    "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
-    is_sdo=True
-)
-class TestClass:
-    pass
-
-
-def test_binary_list():
-    return TestClass(prop_name=["AREi", "7t3M"])
-
-@stix2.CustomObject(
-        "test2-object", [
-            ("prop_name", stix2.properties.ListProperty(
-                stix2.properties.HexProperty()
-            ))
-        ],
-        "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
-        is_sdo=True
-    )
-
-class Test2Class:
-        pass
-
-def test_hex_list():
-    return Test2Class(
-        prop_name=["1122", "fedc"]
-    )
-
-@stix2.CustomObject(
-        "test3-object", [
-            ("prop_name",
-                 stix2.properties.DictionaryProperty(
-                     valid_types=[
-                         stix2.properties.IntegerProperty,
-                         stix2.properties.FloatProperty,
-                         stix2.properties.StringProperty
-                     ]
-                 )
-             )
-        ],
-        "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
-        is_sdo=True
-    )
-class Test3Class:
-    pass
-
-
-def test_dictionary():
-    return Test3Class(
-        prop_name={"a": 1, "b": 2.3, "c": "foo"}
-    )
+# @stix2.CustomObject(
+#     "test-object", [
+#         ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
+#     ],
+#     "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
+#     is_sdo=True
+# )
+# class TestClass:
+#     pass
+#
+#
+# def test_binary_list():
+#     return TestClass(prop_name=["AREi", "7t3M"])
+#
+# @stix2.CustomObject(
+#         "test2-object", [
+#             ("prop_name", stix2.properties.ListProperty(
+#                 stix2.properties.HexProperty()
+#             ))
+#         ],
+#         "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
+#         is_sdo=True
+#     )
+#
+# class Test2Class:
+#         pass
+#
+# def test_hex_list():
+#     return Test2Class(
+#         prop_name=["1122", "fedc"]
+#     )
+#
+# @stix2.CustomObject(
+#         "test3-object", [
+#             ("prop_name",
+#                  stix2.properties.DictionaryProperty(
+#                      valid_types=[
+#                          stix2.properties.IntegerProperty,
+#                          stix2.properties.FloatProperty,
+#                          stix2.properties.StringProperty
+#                      ]
+#                  )
+#              )
+#         ],
+#         "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
+#         is_sdo=True
+#     )
+# class Test3Class:
+#     pass
+#
+#
+# def test_dictionary():
+#     return Test3Class(
+#         prop_name={"a": 1, "b": 2.3, "c": "foo"}
+#     )
 
 
 def main():
@@ -263,17 +263,17 @@ def main():
         store.sink.generate_stix_schema()
         store.sink.clear_tables()
 
-        td = test_dictionary()
+        # td = test_dictionary()
 
-        store.add(td)
+        # store.add(td)
 
-        th = test_hex_list()
+        # th = test_hex_list()
 
         # store.add(th)
 
-        tb = test_binary_list()
+        # tb = test_binary_list()
 
-        store.add(tb)
+        # store.add(tb)
 
 
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index fc3e6b5d..05b0e97d 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -8,8 +8,8 @@
 from stix2.datastore.relational_db.add_method import add_method
 from stix2.datastore.relational_db.utils import (
     SCO_COMMON_PROPERTIES, SDO_COMMON_PROPERTIES, canonicalize_table_name,
-    determine_column_name, determine_sql_type_from_class, flat_classes,
-    get_stix_object_classes, schema_for,
+    determine_column_name, determine_sql_type_from_stix, flat_classes,
+    get_stix_object_classes,
 )
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
@@ -19,7 +19,7 @@
     TimestampProperty, TypeProperty,
 )
 from stix2.v21.base import _Extension
-from stix2.v21.common import KillChainPhase
+from stix2.v21.common import KillChainPhase, GranularMarking
 
 
 def aux_table_property(prop, name, core_properties):
@@ -40,9 +40,10 @@ def derive_column_name(prop):
         return "value"
 
 
-def create_object_markings_refs_table(metadata, sco_or_sdo):
+def create_object_markings_refs_table(metadata, db_backend, sco_or_sdo):
     return create_ref_table(
         metadata,
+        db_backend,
         {"marking-definition"},
         "object_marking_refs_" + sco_or_sdo,
         "common.core_" + sco_or_sdo + ".id",
@@ -51,12 +52,12 @@ def create_object_markings_refs_table(metadata, sco_or_sdo):
     )
 
 
-def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_name, auth_type=0):
+def create_ref_table(metadata, db_backend, specifics, table_name, foreign_key_name, schema_name, auth_type=0):
     columns = list()
     columns.append(
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(
                 foreign_key_name,
                 ondelete="CASCADE",
@@ -64,11 +65,11 @@ def create_ref_table(metadata, specifics, table_name, foreign_key_name, schema_n
             nullable=False,
         ),
     )
-    columns.append(ref_column("ref_id", specifics, auth_type))
+    columns.append(ref_column("ref_id", specifics, db_backend, auth_type))
     return Table(table_name, metadata, *columns, schema=schema_name)
 
 
-def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text, level=1):
+def create_hashes_table(name, metadata, db_backend, schema_name, table_name, key_type=Text, level=1):
     columns = list()
     # special case, perhaps because its a single embedded object with hashes, and not a list of embedded object
     # making the parent table's primary key does seem to worl
@@ -88,14 +89,14 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
     columns.append(
         Column(
             "hash_name",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
     columns.append(
         Column(
             "hash_value",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
@@ -108,12 +109,12 @@ def create_hashes_table(name, metadata, schema_name, table_name, key_type=Text,
     )
 
 
-def create_kill_chain_phases_table(name, metadata, schema_name, table_name):
+def create_kill_chain_phases_table(name, metadata, db_backend, schema_name, table_name):
     columns = list()
     columns.append(
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(
                 canonicalize_table_name(table_name, schema_name) + ".id",
                 ondelete="CASCADE",
@@ -124,93 +125,120 @@ def create_kill_chain_phases_table(name, metadata, schema_name, table_name):
     columns.append(
         Column(
             "kill_chain_name",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
     columns.append(
         Column(
             "phase_name",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
     return Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name)
 
 
-def create_granular_markings_table(metadata, sco_or_sdo):
-    return Table(
-        "granular_marking_" + sco_or_sdo,
-        metadata,
+def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
+    tables = list()
+    columns = [
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
             nullable=False,
         ),
-        Column("lang", Text),
+        Column("lang", db_backend.determine_sql_type_for_string_property()),
         Column(
             "marking_ref",
-            Text,
+            db_backend.determine_sql_type_for_reference_property(),
             CheckConstraint(
-                "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
+                "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
+                # noqa: E131
             ),
-        ),
-        Column(
-            "selectors",
-            ARRAY(Text),
-            CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
-            nullable=False,
-        ),
-        CheckConstraint(
-            """(lang IS NULL AND marking_ref IS NOT NULL)
-               OR
-               (lang IS NOT NULL AND marking_ref IS NULL)""",
-        ),
-        schema="common",
-    )
+        )
+    ]
+    if db_backend.array_allowed():
+        columns.append(
+            Column(
+                "selectors",
+                ARRAY(Text),
+                CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
+                nullable=False,
+            ))
+    else:
+        table_name = "granular_marking_" + sco_or_sdo
+        schema_name = determine_sql_type_from_stix(GranularMarking, db_backend)
+        columns = [
+            Column(
+                "id",
+                db_backend.determine_sql_type_for_key_as_id(),
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+                nullable=False,
+            ),
+            Column(
+                "selector",
+                db_backend.determine_sql_type_for_string_property(),
+                nullable=False,
+            )
+        ]
+        tables.append(Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name))
+    tables.append(Table(
+                    "granular_marking_" + sco_or_sdo,
+                    metadata,
+                    *columns,
+                    CheckConstraint(
+                        """(lang IS NULL AND marking_ref IS NOT NULL)
+                           OR
+                           (lang IS NOT NULL AND marking_ref IS NULL)""",
+                    ),
+                    schema="common"))
+    return tables
 
 
-def create_external_references_tables(metadata):
+def create_external_references_tables(metadata, db_backend):
     columns = [
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey("common.core_sdo" + ".id", ondelete="CASCADE"),
             CheckConstraint(
                 "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
             ),
         ),
-        Column("source_name", Text),
-        Column("description", Text),
-        Column("url", Text),
-        Column("external_id", Text),
+        Column("source_name", db_backend.determine_sql_type_for_string_property()),
+        Column("description", db_backend.determine_sql_type_for_string_property()),
+        Column("url", db_backend.determine_sql_type_for_string_property()),
+        Column("external_id", db_backend.determine_sql_type_for_string_property()),
         # all such keys are generated using the global sequence.
-        Column("hash_ref_id", Integer, primary_key=True, autoincrement=False),
+        Column("hash_ref_id", db_backend.determine_sql_type_for_key_as_int(), primary_key=True, autoincrement=False),
     ]
     return [
         Table("external_references", metadata, *columns, schema="common"),
-        create_hashes_table("hashes", metadata, "common", "external_references", Integer),
+        create_hashes_table("hashes", metadata, db_backend, "common", "external_references", Integer),
     ]
 
 
-def create_core_table(metadata, schema_name):
+def create_core_table(metadata, db_backend, schema_name):
     columns = [
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             CheckConstraint(
                 "id ~ '^[a-z][a-z0-9-]+[a-z0-9]--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
             ),
             primary_key=True,
         ),
-        Column("spec_version", Text, default="2.1"),
+        Column("spec_version", db_backend.determine_sql_type_for_string_property(), default="2.1"),
     ]
     if schema_name == "sdo":
         sdo_columns = [
             Column(
                 "created_by_ref",
-                Text,
+                db_backend.determine_sql_type_for_reference_property(),
                 CheckConstraint(
                     "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",   # noqa: E131
                 ),
@@ -219,12 +247,13 @@ def create_core_table(metadata, schema_name):
             Column("modified", TIMESTAMP(timezone=True)),
             Column("revoked", Boolean),
             Column("confidence", Integer),
-            Column("lang", Text),
+            Column("lang", db_backend.determine_sql_type_for_string_property()),
             Column("labels", ARRAY(Text)),
         ]
         columns.extend(sdo_columns)
     else:
-        columns.append(Column("defanged", Boolean, default=False)),
+        columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
+
     return Table(
         "core_" + schema_name,
         metadata,
@@ -234,60 +263,60 @@ def create_core_table(metadata, schema_name):
 
 
 @add_method(Property)
-def determine_sql_type(self):  # noqa: F811
+def determine_sql_type(self, db_backend):  # noqa: F811
     pass
 
 
 @add_method(KillChainPhase)
-def determine_sql_type(self):  # noqa: F811
-    return None
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_kill_chain_phase()
 
 
 @add_method(BinaryProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Text
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_binary_property()
 
 
 @add_method(BooleanProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Boolean
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_boolean_property()
 
 
 @add_method(FloatProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Float
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_float_property()
 
 
 @add_method(HexProperty)
-def determine_sql_type(self):  # noqa: F811
-    return LargeBinary
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_hex_property()
 
 
 @add_method(IntegerProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Integer
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_integer_property()
 
 
 @add_method(ReferenceProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Text
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_reference_property()
 
 
 @add_method(StringProperty)
-def determine_sql_type(self):  # noqa: F811
-    return Text
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_string_property()
 
 
 @add_method(TimestampProperty)
-def determine_sql_type(self):  # noqa: F811
-    return TIMESTAMP(timezone=True)
+def determine_sql_type(self, db_backend):  # noqa: F811
+    return db_backend.determine_sql_type_for_timestamp_property()
 
 
 # ----------------------------- generate_table_information methods ----------------------------
 
 @add_method(KillChainPhase)
 def generate_table_information(  # noqa: F811
-        self, name, metadata, schema_name, table_name, is_extension=False, is_list=False,
+        self, name, db_backend, metadata, schema_name, table_name, is_extension=False, is_list=False,
         **kwargs,
 ):
     level = kwargs.get("level")
@@ -298,15 +327,15 @@ def generate_table_information(  # noqa: F811
 
 
 @add_method(Property)
-def generate_table_information(self, name, **kwargs):   # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):   # noqa: F811
     pass
 
 
 @add_method(BinaryProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Text,
+        self.determine_sql_type(db_backend),
         CheckConstraint(
             # this regular expression might accept or reject some legal base64 strings
             f"{name} ~  " + "'^[-A-Za-z0-9+/]*={0,3}$'",
@@ -316,30 +345,30 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 
 @add_method(BooleanProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Boolean,
+        self.determine_sql_type(db_backend),
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
 @add_method(DictionaryProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     columns = list()
 
     columns.append(
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
         ),
     )
     columns.append(
         Column(
             "name",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
@@ -350,7 +379,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
                     Column(
                         "value",
                         # its a class
-                        determine_sql_type_from_class(self.valid_types[0]),
+                        determine_sql_type_from_stix(self.valid_types[0], db_backend),
                         nullable=False,
                     ),
                 )
@@ -360,13 +389,13 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
                     Column(
                         "value",
                         # its an instance, not a class
-                        ARRAY(contained_class.determine_sql_type()),
+                        ARRAY(contained_class.determine_sql_type(db_backend)),
                         nullable=False,
                     ),
                 )
         else:
             for column_type in self.valid_types:
-                sql_type = determine_sql_type_from_class(column_type)
+                sql_type = determine_sql_type_from_stix(column_type, db_backend)
                 columns.append(
                     Column(
                         determine_column_name(column_type),
@@ -377,7 +406,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
         columns.append(
             Column(
                 "value",
-                Text,
+                db_backend.determine_sql_type_for_string_property(),
                 nullable=False,
             ),
         )
@@ -392,24 +421,21 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
     ]
 
 
-
-
-
 @add_method(EmbeddedObjectProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, is_extension=False, is_list=False, **kwargs):  # noqa: F811
     level = kwargs.get("level")
     return generate_object_table(
-        self.type, metadata, schema_name, table_name, is_extension, True, is_list,
+        self.type, db_backend, metadata, schema_name, table_name, is_extension, True, is_list,
         parent_table_name=table_name, level=level+1 if is_list else level,
     )
 
 
 @add_method(EnumProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     enum_re = "|".join(self.allowed)
     return Column(
         name,
-        Text,
+        self.determine_sql_type(db_backend),
         CheckConstraint(
             f"{name} ~ '^{enum_re}$'",
         ),
@@ -418,12 +444,12 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 
 @add_method(ExtensionsProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     columns = list()
     columns.append(
         Column(
             "id",
-            Text,
+            db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(canonicalize_table_name(table_name, schema_name) + ".id", ondelete="CASCADE"),
             nullable=False,
         ),
@@ -431,7 +457,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
     columns.append(
         Column(
             "ext_table_name",
-            Text,
+            db_backend.determine_sql_type_for_string_property(),
             nullable=False,
         ),
     )
@@ -439,17 +465,17 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
 
 
 @add_method(FloatProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Float,
+        self.determine_sql_type(db_backend),
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
 @add_method(HashesProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, is_extension=False, **kwargs):  # noqa: F811
     level = kwargs.get("level")
     if kwargs.get("is_embedded_object"):
         if not kwargs.get("is_list") or level == 0:
@@ -462,6 +488,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
         create_hashes_table(
             name,
             metadata,
+            db_backend,
             schema_name,
             table_name,
             key_type=key_type,
@@ -471,21 +498,19 @@ def generate_table_information(self, name, metadata, schema_name, table_name, is
 
 
 @add_method(HexProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        LargeBinary,
+        db_backend.determine_sql_type_for_hex_property(),
         nullable=not self.required,
     )
 
 
 @add_method(IDProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     schema_name = kwargs.get('schema_name')
-    if schema_name in ["sro", "common"]:
-        # sro, smo common properties are the same as sdo's
-        schema_name = "sdo"
     table_name = kwargs.get("table_name")
+    core_table = kwargs.get("core_table")
     # if schema_name == "common":
     #     return Column(
     #         name,
@@ -498,10 +523,13 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
     #         nullable=not (self.required),
     #     )
     # else:
-    foreign_key_column = f"common.core_{schema_name}.id"
+    if schema_name:
+        foreign_key_column = f"common.core_{core_table}.id"
+    else:
+        foreign_key_column = f"core_{core_table}.id"
     return Column(
         name,
-        Text,
+        db_backend.determine_sql_type_for_key_as_id(),
         ForeignKey(foreign_key_column, ondelete="CASCADE"),
         CheckConstraint(
             f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
@@ -511,63 +539,28 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
         nullable=not (self.required),
     )
 
-    return Column(
-        name,
-        Text,
-        ForeignKey(foreign_key_column, ondelete="CASCADE"),
-        CheckConstraint(
-            f"{name} ~ '^{table_name}" + "--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",  # noqa: E131
-        ),
-        primary_key=True,
-        nullable=not (self.required),
-    )
-
 
 @add_method(IntegerProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Integer,
+        self.determine_sql_type(db_backend),
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
 @add_method(ListProperty)
-def generate_table_information(self, name, metadata, schema_name, table_name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     is_extension = kwargs.get('is_extension')
     tables = list()
-    if isinstance(self.contained, ReferenceProperty):
-        return [
-            create_ref_table(
-                metadata,
-                self.contained.specifics,
-                canonicalize_table_name(table_name + "_" + name),
-                canonicalize_table_name(table_name, schema_name) + ".id",
-                schema_name,
-            ),
-        ]
-    elif isinstance(self.contained, EnumProperty):
+    # handle more complext embedded object before deciding if the ARRAY type is usable
+    if isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(
             Column(
                 "id",
-                Text,
-                ForeignKey(
-                    canonicalize_table_name(table_name, schema_name) + ".id",
-                    ondelete="CASCADE",
-                ),
-                nullable=False,
-            ),
-        )
-        columns.append(self.contained.generate_table_information(name))
-        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
-    elif isinstance(self.contained, EmbeddedObjectProperty):
-        columns = list()
-        columns.append(
-            Column(
-                "id",
-                Text,
+                db_backend.determine_sql_type_for_key_as_id(),
                 ForeignKey(
                     canonicalize_table_name(table_name, schema_name) + ".id",
                     ondelete="CASCADE",
@@ -577,7 +570,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
         columns.append(
             Column(
                 "ref_id",
-                Integer,
+                db_backend.determine_sql_type_for_key_as_int(),
                 primary_key=True,
                 nullable=False,
                 # all such keys are generated using the global sequence.
@@ -588,9 +581,11 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
         tables.extend(
             self.contained.generate_table_information(
                 name,
+                db_backend,
                 metadata,
                 schema_name,
-                canonicalize_table_name(table_name + "_" + name, None),  # if sub_table_needed else canonicalize_table_name(table_name, None),
+                canonicalize_table_name(table_name + "_" + name, None),
+                # if sub_table_needed else canonicalize_table_name(table_name, None),
                 is_extension,
                 parent_table_name=table_name,
                 is_list=True,
@@ -598,12 +593,41 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
             ),
         )
         return tables
+    elif isinstance(self.contained, ReferenceProperty):
+        return [
+            create_ref_table(
+                metadata,
+                db_backend,
+                self.contained.specifics,
+                canonicalize_table_name(table_name + "_" + name),
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                schema_name,
+            ),
+        ]
+    elif ((isinstance(self.contained,
+                    (StringProperty, IntegerProperty, FloatProperty)) and not db_backend.array_allowed()) or
+          isinstance(self.contained, EnumProperty)):
+        columns = list()
+        columns.append(
+            Column(
+                "id",
+                self.contained.determine_sql_type(db_backend),
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+                nullable=False,
+            ),
+        )
+        columns.append(self.contained.generate_table_information(name, db_backend))
+        tables.append(Table(canonicalize_table_name(table_name + "_" + name), metadata, *columns, schema=schema_name))
+
     elif self.contained == KillChainPhase:
-        tables.append(create_kill_chain_phases_table(name, metadata, schema_name, table_name))
+        tables.append(create_kill_chain_phases_table(name, metadata, db_backend, schema_name, table_name))
         return tables
     else:
         if isinstance(self.contained, Property):
-            sql_type = self.contained.determine_sql_type()
+            sql_type = self.contained.determine_sql_type(db_backend)
             if sql_type:
                 return Column(
                     name,
@@ -612,7 +636,7 @@ def generate_table_information(self, name, metadata, schema_name, table_name, **
                 )
 
 
-def ref_column(name, specifics, auth_type=0):
+def ref_column(name, specifics, db_backend, auth_type=0):
     if specifics:
         types = "|".join(specifics)
         if auth_type == 0:
@@ -627,41 +651,41 @@ def ref_column(name, specifics, auth_type=0):
                     f"(NOT({name} ~ '^({types})')) AND ({name} ~ " +
                     "'--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$')",
                 )
-        return Column(name, Text, constraint)
+        return Column(name, db_backend.determine_sql_type_for_reference_property(), constraint)
     else:
         return Column(
             name,
-            Text,
+            db_backend.determine_sql_type_for_reference_property(),
             nullable=False,
         )
 
 
 @add_method(ObjectReferenceProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     table_name = kwargs.get('table_name')
     raise ValueError(f"Property {name} in {table_name} is of type ObjectReferenceProperty, which is for STIX 2.0 only")
 
 
 @add_method(ReferenceProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
-    return ref_column(name, self.specifics, self.auth_type)
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
+    return ref_column(name, self.specifics, db_backend, self.auth_type)
 
 
 @add_method(StringProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Text,
+        db_backend.determine_sql_type_for_string_property(),
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
 @add_method(TimestampProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        TIMESTAMP(timezone=True),
+        self.determine_sql_type(db_backend),
         # CheckConstraint(
         #     f"{name} ~ '^{enum_re}$'"
         # ),
@@ -670,17 +694,17 @@ def generate_table_information(self, name, **kwargs):  # noqa: F811
 
 
 @add_method(TypeProperty)
-def generate_table_information(self, name, **kwargs):  # noqa: F811
+def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
     return Column(
         name,
-        Text,
+        db_backend.determine_sql_type_for_string_property(),
         nullable=not self.required,
         default=self._fixed_value if hasattr(self, "_fixed_value") else None,
     )
 
 
 def generate_object_table(
-    stix_object_class, metadata, schema_name, foreign_key_name=None,
+    stix_object_class, db_backend, metadata, schema_name, foreign_key_name=None,
     is_extension=False, is_embedded_object=False, is_list=False, parent_table_name=None, level=0,
 ):
     properties = stix_object_class._properties
@@ -704,11 +728,17 @@ def generate_object_table(
         core_properties = list()
     columns = list()
     tables = list()
+    if schema_name == "sco":
+        core_table = "sco"
+    else:
+        # sro, smo common properties are the same as sdo's
+        core_table = "sdo"
     for name, prop in properties.items():
         # type is never a column since it is implicit in the table
         if (name == 'id' or name not in core_properties) and name != 'type':
             col = prop.generate_table_information(
                 name,
+                db_backend,
                 metadata=metadata,
                 schema_name=schema_name,
                 table_name=table_name,
@@ -717,6 +747,7 @@ def generate_object_table(
                 is_list=is_list,
                 level=level,
                 parent_table_name=parent_table_name,
+                core_table=core_table,
             )
             if col is not None and isinstance(col, Column):
                 columns.append(col)
@@ -726,7 +757,7 @@ def generate_object_table(
         columns.append(
             Column(
                 "id",
-                Text,
+                db_backend.determine_sql_type_for_key_as_id(),
                 # no Foreign Key because it could be for different tables
                 primary_key=True,
             ),
@@ -736,7 +767,7 @@ def generate_object_table(
             if is_extension and not is_embedded_object:
                 column = Column(
                     "id",
-                    Text,
+                    db_backend.determine_sql_type_for_key_as_id(),
                     ForeignKey(
                         canonicalize_table_name(foreign_key_name, schema_name) + ".id",
                         ondelete="CASCADE",
@@ -745,7 +776,7 @@ def generate_object_table(
             elif is_embedded_object:
                 column = Column(
                     "id",
-                    Integer if is_list else Text,
+                    db_backend.determine_sql_type_for_key_as_int() if is_list else db_backend.determine_sql_type_for_key_as_id(),
                     ForeignKey(
                         canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if is_list else ".id"),
                         ondelete="CASCADE",
@@ -756,7 +787,7 @@ def generate_object_table(
         elif level > 0 and is_embedded_object:
             column = Column(
                 "id",
-                Integer if (is_embedded_object and is_list) else Text,
+                db_backend.determine_sql_type_for_key_as_int() if (is_embedded_object and is_list) else db_backend.determine_sql_type_for_key_as_id(),
                 ForeignKey(
                     canonicalize_table_name(foreign_key_name, schema_name) + (".ref_id" if (is_embedded_object and is_list) else ".id"),
                     ondelete="CASCADE",
@@ -767,7 +798,7 @@ def generate_object_table(
         else:
             column = Column(
                 "id",
-                Text,
+                db_backend.determine_sql_type_for_key_as_id(),
                 ForeignKey(
                     canonicalize_table_name(foreign_key_name, schema_name) + ".id",
                     ondelete="CASCADE",
@@ -783,20 +814,20 @@ def generate_object_table(
     return tables
 
 
-def create_core_tables(metadata):
+def create_core_tables(metadata, db_backend):
     tables = [
-        create_core_table(metadata, "sdo"),
-        create_granular_markings_table(metadata, "sdo"),
-        create_core_table(metadata, "sco"),
-        create_granular_markings_table(metadata, "sco"),
-        create_object_markings_refs_table(metadata, "sdo"),
-        create_object_markings_refs_table(metadata, "sco"),
+        create_core_table(metadata, db_backend, "sdo"),
+        create_granular_markings_table(metadata, db_backend, "sdo"),
+        create_core_table(metadata, db_backend, "sco"),
+        create_granular_markings_table(metadata, db_backend, "sco"),
+        create_object_markings_refs_table(metadata, db_backend, "sdo"),
+        create_object_markings_refs_table(metadata, db_backend, "sco"),
     ]
-    tables.extend(create_external_references_tables(metadata))
+    tables.extend(create_external_references_tables(metadata, db_backend))
     return tables
 
 
-def create_table_objects(metadata, stix_object_classes):
+def create_table_objects(metadata, db_backend, stix_object_classes):
     if stix_object_classes:
         # If classes are given, allow some flexibility regarding lists of
         # classes vs single classes
@@ -806,16 +837,17 @@ def create_table_objects(metadata, stix_object_classes):
         # If no classes given explicitly, discover them automatically
         stix_object_classes = get_stix_object_classes()
 
-    tables = create_core_tables(metadata)
+    tables = create_core_tables(metadata, db_backend)
 
     for stix_class in stix_object_classes:
 
-        schema_name = schema_for(stix_class)
+        schema_name = db_backend.schema_for(stix_class)
         is_extension = issubclass(stix_class, _Extension)
 
         tables.extend(
             generate_object_table(
                 stix_class,
+                db_backend,
                 metadata,
                 schema_name,
                 is_extension=is_extension,
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 5d06de7b..76a4268b 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -136,23 +136,25 @@ def is_class_or_instance(cls_or_inst, cls):
     return cls_or_inst == cls or isinstance(cls_or_inst, cls)
 
 
-def determine_sql_type_from_class(cls_or_inst):  # noqa: F811
+def determine_sql_type_from_stix(cls_or_inst, db_backend):  # noqa: F811
     if is_class_or_instance(cls_or_inst, BinaryProperty):
-        return LargeBinary
+        return db_backend.determine_sql_type_for_binary_property()
     elif is_class_or_instance(cls_or_inst, BooleanProperty):
-        return Boolean
-    elif is_class_or_instance(cls_or_inst, FloatProperty ):
-        return Float
+        return db_backend.determine_sql_type_for_boolean_property()
+    elif is_class_or_instance(cls_or_inst, FloatProperty):
+        return db_backend.determine_sql_type_for_float_property()
     elif is_class_or_instance(cls_or_inst, HexProperty):
-        return LargeBinary
+        return db_backend.determine_sql_type_for_hex_property()
     elif is_class_or_instance(cls_or_inst, IntegerProperty):
-        return Integer
-    elif is_class_or_instance(cls_or_inst, StringProperty) or is_class_or_instance(cls_or_inst, ReferenceProperty):
-        return Text
+        return db_backend.determine_sql_type_for_integer_property()
+    elif is_class_or_instance(cls_or_inst, StringProperty):
+        return db_backend.determine_sql_type_for_integer_property()
+    elif is_class_or_instance(cls_or_inst, ReferenceProperty):
+        db_backend.determine_sql_type_for_reference_property()
     elif is_class_or_instance(cls_or_inst, TimestampProperty):
-        return TIMESTAMP(timezone=True)
+        return db_backend.determine_sql_type_for_timestamp_property()
     elif is_class_or_instance(cls_or_inst, Property):
-        return Text
+        return db_backend.determine_sql_type_for_integer_property()
 
 
 def determine_column_name(cls_or_inst):  # noqa: F811

From 507914e26ac636f5f0f62f0fbfe2646dab158094 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 7 Nov 2024 17:07:57 -0500
Subject: [PATCH 122/132] more ARRAY

---
 .../database_backend_base.py                  | 21 ++---
 .../database_backends/postgres_backend.py     | 18 ++++-
 .../datastore/relational_db/table_creation.py | 80 +++++++++++--------
 3 files changed, 68 insertions(+), 51 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index 5b65119b..a3e74240 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -23,7 +23,7 @@ def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any)
         self.database_connection = create_engine(database_connection_url)
 
     def _create_schemas(self):
-        return
+        pass
 
     @staticmethod
     def _determine_schema_name(stix_object):
@@ -35,7 +35,11 @@ def _create_database(self):
         create_database(self.database_connection.url)
         self.database_exists = database_exists(self.database_connection.url)
 
-    def schema_for(self, stix_class):
+    def schema_for(stix_class):
+        return ""
+
+    @staticmethod
+    def schema_for_core():
         return ""
 
     @staticmethod
@@ -46,10 +50,6 @@ def determine_sql_type_for_property():  # noqa: F811
     def determine_sql_type_for_kill_chain_phase():  # noqa: F811
         return None
 
-    @staticmethod
-    def determine_sql_type_for_binary_property():  # noqa: F811
-        return Text
-
     @staticmethod
     def determine_sql_type_for_boolean_property():  # noqa: F811
         return Boolean
@@ -58,10 +58,6 @@ def determine_sql_type_for_boolean_property():  # noqa: F811
     def determine_sql_type_for_float_property():  # noqa: F811
         return Float
 
-    @staticmethod
-    def determine_sql_type_for_hex_property():  # noqa: F811
-        return LargeBinary
-
     @staticmethod
     def determine_sql_type_for_integer_property():  # noqa: F811
         return Integer
@@ -74,15 +70,10 @@ def determine_sql_type_for_reference_property():  # noqa: F811
     def determine_sql_type_for_string_property():  # noqa: F811
         return Text
 
-    @staticmethod
-    def determine_sql_type_for_timestamp_property():  # noqa: F811
-        return TIMESTAMP(timezone=True)
-
     @staticmethod
     def determine_sql_type_for_key_as_int():  # noqa: F811
         return Integer
 
-
     @staticmethod
     def determine_sql_type_for_key_as_id():  # noqa: F811
         return Text
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index 08ca3c9e..f88dbb10 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -50,9 +50,25 @@ def _determine_schema_name(stix_object):
         elif isinstance(stix_object, _MetaObject):
             return "common"
 
-    def schema_for(self, stix_class):
+    @staticmethod
+    def schema_for(stix_class):
         return schema_for(stix_class)
 
+    @staticmethod
+    def schema_for_core():
+        return "common"
+
+    @staticmethod
+    def determine_sql_type_for_binary_property():  # noqa: F811
+        return Text
+
+    @staticmethod
+    def determine_sql_type_for_hex_property():  # noqa: F811
+        return LargeBinary
+
+    @staticmethod
+    def determine_sql_type_for_timestamp_property():  # noqa: F811
+        return TIMESTAMP(timezone=True)
 
     @staticmethod
     def array_allowed():
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 05b0e97d..1df146c4 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -32,6 +32,33 @@ def aux_table_property(prop, name, core_properties):
         return False
 
 
+def create_array_column(property_name, contained_sql_type):
+    return Column(property_name,
+                  ARRAY(contained_sql_type),
+                  CheckConstraint(f"array_length({property_name}, 1) IS NOT NULL"),
+                  nullable=False)
+
+
+def create_array_child_table(metadata, db_backend, table_name, property_name, contained_sql_type):
+        schema_name = determine_sql_type_from_stix(GranularMarking, db_backend)
+        columns = [
+            Column(
+                "id",
+                db_backend.determine_sql_type_for_key_as_id(),
+                ForeignKey(
+                    canonicalize_table_name(table_name, schema_name) + ".id",
+                    ondelete="CASCADE",
+                ),
+                nullable=False,
+            ),
+            Column(
+                property_name,
+                contained_sql_type,
+                nullable=False,
+            )
+        ]
+        return Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name)
+
 def derive_column_name(prop):
     contained_property = prop.contained
     if isinstance(contained_property, ReferenceProperty):
@@ -159,33 +186,14 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
         )
     ]
     if db_backend.array_allowed():
-        columns.append(
-            Column(
-                "selectors",
-                ARRAY(Text),
-                CheckConstraint("array_length(selectors, 1) IS NOT NULL"),
-                nullable=False,
-            ))
+        columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property()))
     else:
-        table_name = "granular_marking_" + sco_or_sdo
-        schema_name = determine_sql_type_from_stix(GranularMarking, db_backend)
-        columns = [
-            Column(
-                "id",
-                db_backend.determine_sql_type_for_key_as_id(),
-                ForeignKey(
-                    canonicalize_table_name(table_name, schema_name) + ".id",
-                    ondelete="CASCADE",
-                ),
-                nullable=False,
-            ),
-            Column(
-                "selector",
-                db_backend.determine_sql_type_for_string_property(),
-                nullable=False,
-            )
-        ]
-        tables.append(Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name))
+        tables.append(create_array_child_table(metadata,
+                                               db_backend,
+                                               "granular_marking_" + sco_or_sdo,
+                                               "selector",
+                                               db_backend.determine_sql_type_for_string_property()))
+
     tables.append(Table(
                     "granular_marking_" + sco_or_sdo,
                     metadata,
@@ -223,6 +231,7 @@ def create_external_references_tables(metadata, db_backend):
 
 
 def create_core_table(metadata, db_backend, schema_name):
+    tables = list()
     columns = [
         Column(
             "id",
@@ -248,18 +257,20 @@ def create_core_table(metadata, db_backend, schema_name):
             Column("revoked", Boolean),
             Column("confidence", Integer),
             Column("lang", db_backend.determine_sql_type_for_string_property()),
-            Column("labels", ARRAY(Text)),
         ]
         columns.extend(sdo_columns)
+        if db_backend.array_allowed():
+            columns.append(create_array_column("labels", db_backend.determine_sql_type_for_string_property())),
     else:
         columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
 
-    return Table(
-        "core_" + schema_name,
-        metadata,
-        *columns,
-        schema="common",
-    )
+    tables = [
+        Table("core_" + schema_name,
+              metadata,
+              *columns,
+              schema=db_backend.schema_for_core())
+    ]
+    return tables
 
 
 @add_method(Property)
@@ -604,8 +615,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
                 schema_name,
             ),
         ]
-    elif ((isinstance(self.contained,
-                    (StringProperty, IntegerProperty, FloatProperty)) and not db_backend.array_allowed()) or
+    elif ((isinstance(self.contained, (StringProperty, IntegerProperty, FloatProperty)) and not db_backend.array_allowed()) or
           isinstance(self.contained, EnumProperty)):
         columns = list()
         columns.append(

From a84867ccf1a89db3ec6b2bd425fab579220bcded Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Fri, 8 Nov 2024 08:33:32 -0500
Subject: [PATCH 123/132] flaky

---
 stix2/datastore/relational_db/query.py        | 28 +++----
 .../relational_db/relational_db_testing.py    | 28 +++----
 stix2/properties.py                           |  8 +-
 .../test/v21/test_datastore_relational_db.py  | 73 ++++++++++---------
 stix2/test/v21/test_properties.py             |  7 +-
 5 files changed, 72 insertions(+), 72 deletions(-)

diff --git a/stix2/datastore/relational_db/query.py b/stix2/datastore/relational_db/query.py
index 50bac5e0..f223852b 100644
--- a/stix2/datastore/relational_db/query.py
+++ b/stix2/datastore/relational_db/query.py
@@ -109,7 +109,7 @@ def _read_hashes(fk_id, hashes_table, conn):
     :return: The hashes as a dict, or None if no hashes were found
     """
     stmt = sa.select(hashes_table.c.hash_name, hashes_table.c.hash_value).where(
-        hashes_table.c.id == fk_id
+        hashes_table.c.id == fk_id,
     )
 
     results = conn.execute(stmt)
@@ -242,7 +242,7 @@ def _read_kill_chain_phases(stix_id, type_table, metadata, conn):
     kill_chain_phases_table = metadata.tables[type_table.fullname + "_kill_chain_phase"]
     stmt = sa.select(
         kill_chain_phases_table.c.kill_chain_name,
-        kill_chain_phases_table.c.phase_name
+        kill_chain_phases_table.c.phase_name,
     ).where(kill_chain_phases_table.c.id == stix_id)
 
     kill_chain_phases = conn.execute(stmt).mappings().all()
@@ -268,9 +268,9 @@ def _read_dictionary_property(stix_id, type_table, prop_name, prop_instance, met
 
     if len(prop_instance.valid_types) == 1:
         stmt = sa.select(
-            dict_table.c.name, dict_table.c.value
+            dict_table.c.name, dict_table.c.value,
         ).where(
-            dict_table.c.id == stix_id
+            dict_table.c.id == stix_id,
         )
 
         results = conn.execute(stmt)
@@ -291,7 +291,7 @@ def _read_dictionary_property(stix_id, type_table, prop_name, prop_instance, met
             if first_non_null_value is None:
                 raise DataSourceError(
                     f'In dictionary table {dict_table.fullname}, key "{key}"'
-                    " did not map to a non-null value"
+                    " did not map to a non-null value",
                 )
 
             dict_value[key] = first_non_null_value
@@ -320,7 +320,7 @@ def _read_embedded_object(obj_id, parent_table, embedded_type, metadata, conn):
 
     embedded_table_name = canonicalize_table_name(
         f"{parent_table.name}_{embedded_type.__name__}",
-        parent_table.schema
+        parent_table.schema,
     )
     embedded_table = metadata.tables[embedded_table_name]
 
@@ -346,7 +346,7 @@ def _read_embedded_object(obj_id, parent_table, embedded_type, metadata, conn):
                     prop_instance,
                     embedded_table,
                     metadata,
-                    conn
+                    conn,
                 )
 
                 if prop_value is not None:
@@ -375,7 +375,7 @@ def _read_embedded_object_list(fk_id, join_table, embedded_type, metadata, conn)
 
     embedded_table_name = canonicalize_table_name(
         f"{join_table.name}_{embedded_type.__name__}",
-        join_table.schema
+        join_table.schema,
     )
     embedded_table = metadata.tables[embedded_table_name]
 
@@ -394,7 +394,7 @@ def _read_embedded_object_list(fk_id, join_table, embedded_type, metadata, conn)
                     prop_instance,
                     embedded_table,
                     metadata,
-                    conn
+                    conn,
                 )
 
                 if prop_value is not None:
@@ -450,7 +450,7 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
                 join_table,
                 prop_instance.contained.type,
                 metadata,
-                conn
+                conn,
             )
 
         elif inspect.isclass(prop_instance.contained) and issubclass(prop_instance.contained, stix2.KillChainPhase):
@@ -459,7 +459,7 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
         else:
             raise DataSourceError(
                 f'Not implemented: read "{prop_name}" property value'
-                f" of type list-of {prop_instance.contained}"
+                f" of type list-of {prop_instance.contained}",
             )
 
     elif isinstance(prop_instance, stix2.properties.HashesProperty):
@@ -482,13 +482,13 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
             obj_table,
             prop_instance.type,
             metadata,
-            conn
+            conn,
         )
 
     else:
         raise DataSourceError(
             f'Not implemented: read "{prop_name}" property value'
-            f" of type {prop_instance.__class__}"
+            f" of type {prop_instance.__class__}",
         )
 
     return prop_value
@@ -577,7 +577,7 @@ def read_object(stix_id, metadata, conn):
                 prop_instance,
                 type_table,
                 metadata,
-                conn
+                conn,
             )
 
             if prop_value is not None:
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index fe35a454..262cf59a 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -1,13 +1,12 @@
 import datetime as dt
 
+from database_backends.postgres_backend import PostgresBackend
 import pytz
 
 import stix2
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 
-from database_backends.postgres_backend import PostgresBackend
-
 directory_stix_object = stix2.Directory(
     path="/foo/bar/a",
     path_enc="latin1",
@@ -107,18 +106,18 @@ def extension_definition_insert():
         extension_types=["property-extension", "new-sdo", "new-sro"],
         object_marking_refs=[
             "marking-definition--caa0d913-5db8-4424-aae0-43e770287d30",
-            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77"
+            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77",
         ],
         granular_markings=[
             {
                 "lang": "en_US",
-                "selectors": ["name", "schema"]
+                "selectors": ["name", "schema"],
             },
             {
                 "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
-                "selectors": ["name", "schema"]
-            }
-        ]
+                "selectors": ["name", "schema"],
+            },
+        ],
     )
 
 
@@ -171,12 +170,13 @@ def kill_chain_test():
              },
         ], )
 
-@stix2.CustomObject('x-custom-type',
+@stix2.CustomObject(
+    'x-custom-type',
         properties=[
             ("phases", stix2.properties.ListProperty(stix2.KillChainPhase)),
-            ("something_else", stix2.properties.IntegerProperty())
-        ]
-    )
+            ("something_else", stix2.properties.IntegerProperty()),
+        ],
+)
 class CustomClass:
     pass
 
@@ -186,10 +186,10 @@ def custom_obj():
         phases=[
             {
                 "kill_chain_name": "chain name",
-                "phase_name": "the phase name"
-            }
+                "phase_name": "the phase name",
+            },
         ],
-        something_else=5
+        something_else=5,
     )
     return obj
 
diff --git a/stix2/properties.py b/stix2/properties.py
index eef757be..43612826 100644
--- a/stix2/properties.py
+++ b/stix2/properties.py
@@ -447,13 +447,13 @@ def _normalize_valid_types(self, valid_types):
                 if not isinstance(prop_instance.contained, simple_types):
                     raise ValueError(
                         "DictionaryProperty does not support lists of type: "
-                        + type(prop_instance.contained).__name__
+                        + type(prop_instance.contained).__name__,
                     )
 
             elif not isinstance(prop_instance, simple_types):
                 raise ValueError(
                     "DictionaryProperty does not support value type: "
-                    + type(prop_instance).__name__
+                    + type(prop_instance).__name__,
                 )
 
             prop_instances.append(prop_instance)
@@ -499,7 +499,7 @@ def clean(self, value, allow_custom=False, strict=False):
                             # Ignore the passed-in value and fix this to True;
                             # we need strict cleaning to disambiguate value
                             # types here.
-                            strict=True
+                            strict=True,
                         )
                     except CustomContentError:
                         # Need to propagate these, not treat as a type error
@@ -524,7 +524,7 @@ def clean(self, value, allow_custom=False, strict=False):
                 else:
                     # clean failed for all properties!
                     raise ValueError(
-                        f"Invalid value: {v!r}"
+                        f"Invalid value: {v!r}",
                     )
 
             # else: no valid types given, so we skip the validity check
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 0c2e3607..5887c066 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -6,8 +6,8 @@
 import pytest
 
 import stix2
-from stix2.datastore.relational_db.relational_db import RelationalDBStore
 from stix2.datastore import DataSourceError
+from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 import stix2.registry
 import stix2.v21
@@ -18,7 +18,7 @@
     _DB_CONNECT_URL,
     True,
     None,
-    False
+    False,
 )
 
 # Artifacts
@@ -705,16 +705,16 @@ class TestClass:
     "string": (stix2.properties.StringProperty(), "test"),
     "timestamp": (
         stix2.properties.TimestampProperty(),
-        datetime.datetime.now(tz=datetime.timezone.utc)
+        datetime.datetime.now(tz=datetime.timezone.utc),
     ),
     "ref": (
         stix2.properties.ReferenceProperty("SDO"),
-        "identity--ec83b570-0743-4179-a5e3-66fd2fae4711"
+        "identity--ec83b570-0743-4179-a5e3-66fd2fae4711",
     ),
     "enum": (
         stix2.properties.EnumProperty(["value1", "value2"]),
-        "value1"
-    )
+        "value1",
+    ),
 }
 
 
@@ -739,8 +739,8 @@ def base_property_value(request):
         "list-dict-of",
         "subobject",
         "list-of-subobject-prop",
-        "list-of-subobject-class"
-    ]
+        "list-of-subobject-class",
+    ],
 )
 def property_variation_value(request, base_property_value):
     """
@@ -755,7 +755,7 @@ class Embedded(stix2.v21._STIXBase21):
         sub-object.
         """
         _properties = {
-            "embedded": base_property
+            "embedded": base_property,
         }
 
     if request.param == "base":
@@ -768,14 +768,14 @@ class Embedded(stix2.v21._STIXBase21):
 
     elif request.param == "dict-of":
         prop_variation = stix2.properties.DictionaryProperty(
-            valid_types=base_property
+            valid_types=base_property,
         )
         # key name doesn't matter here
         prop_variation_value = {"key": prop_value}
 
     elif request.param == "dict-list-of":
         prop_variation = stix2.properties.DictionaryProperty(
-            valid_types=stix2.properties.ListProperty(base_property)
+            valid_types=stix2.properties.ListProperty(base_property),
         )
         # key name doesn't matter here
         prop_variation_value = {"key": [prop_value]}
@@ -798,7 +798,7 @@ class Embedded(stix2.v21._STIXBase21):
     elif request.param == "list-of-subobject-prop":
         # list-of-embedded values via EmbeddedObjectProperty
         prop_variation = stix2.properties.ListProperty(
-            stix2.properties.EmbeddedObjectProperty(Embedded)
+            stix2.properties.EmbeddedObjectProperty(Embedded),
         )
         prop_variation_value = [{"embedded": prop_value}]
 
@@ -831,10 +831,10 @@ def object_variation(request, property_variation_value):
     if request.param == "sdo":
         @stix2.CustomObject(
             "test-object", [
-                ("prop_name", property_instance)
+                ("prop_name", property_instance),
             ],
             ext_id,
-            is_sdo=True
+            is_sdo=True,
         )
         class TestClass:
             pass
@@ -842,10 +842,10 @@ class TestClass:
     elif request.param == "sro":
         @stix2.CustomObject(
             "test-object", [
-                ("prop_name", property_instance)
+                ("prop_name", property_instance),
             ],
             ext_id,
-            is_sdo=False
+            is_sdo=False,
         )
         class TestClass:
             pass
@@ -853,10 +853,10 @@ class TestClass:
     elif request.param == "sco":
         @stix2.CustomObservable(
             "test-object", [
-                ("prop_name", property_instance)
+                ("prop_name", property_instance),
             ],
             ["prop_name"],
-            ext_id
+            ext_id,
         )
         class TestClass:
             pass
@@ -883,7 +883,7 @@ def test_property(object_variation):
         None,
         True,
         True,
-        type(object_variation)
+        type(object_variation),
     )
 
     rdb_store.add(object_variation)
@@ -898,22 +898,23 @@ def test_dictionary_property_complex():
     """
     with _register_object(
         "test-object", [
-            ("prop_name",
-                 stix2.properties.DictionaryProperty(
-                     valid_types=[
-                         stix2.properties.IntegerProperty,
-                         stix2.properties.FloatProperty,
-                         stix2.properties.StringProperty
-                     ]
-                 )
-             )
+            (
+                "prop_name",
+                    stix2.properties.DictionaryProperty(
+                        valid_types=[
+                            stix2.properties.IntegerProperty,
+                            stix2.properties.FloatProperty,
+                            stix2.properties.StringProperty,
+                        ],
+                    ),
+            ),
         ],
         "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
-        is_sdo=True
+        is_sdo=True,
     ) as cls:
 
         obj = cls(
-            prop_name={"a": 1, "b": 2.3, "c": "foo"}
+            prop_name={"a": 1, "b": 2.3, "c": "foo"},
         )
 
         rdb_store = RelationalDBStore(
@@ -922,7 +923,7 @@ def test_dictionary_property_complex():
             None,
             True,
             True,
-            cls
+            cls,
         )
 
         rdb_store.add(obj)
@@ -939,18 +940,18 @@ def test_extension_definition():
         extension_types=["property-extension", "new-sdo", "new-sro"],
         object_marking_refs=[
             "marking-definition--caa0d913-5db8-4424-aae0-43e770287d30",
-            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77"
+            "marking-definition--122a27a0-b96f-46bc-8fcd-f7a159757e77",
         ],
         granular_markings=[
             {
                 "lang": "en_US",
-                "selectors": ["name", "schema"]
+                "selectors": ["name", "schema"],
             },
             {
                 "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
-                "selectors": ["name", "schema"]
-            }
-        ]
+                "selectors": ["name", "schema"],
+            },
+        ],
     )
 
     store.add(obj)
diff --git a/stix2/test/v21/test_properties.py b/stix2/test/v21/test_properties.py
index 2cd42a12..7b9a41e1 100644
--- a/stix2/test/v21/test_properties.py
+++ b/stix2/test/v21/test_properties.py
@@ -15,7 +15,6 @@
 
 from . import constants
 
-
 ID_PROP = IDProperty('my-type', spec_version="2.1")
 MY_ID = 'my-type--232c9d3f-49fc-4440-bb01-607f638778e7'
 
@@ -451,7 +450,7 @@ def test_dictionary_property_values_list():
 
 def test_dictionary_property_ref_custom():
     p = DictionaryProperty(
-        valid_types=ReferenceProperty(valid_types="SDO"), spec_version="2.1"
+        valid_types=ReferenceProperty(valid_types="SDO"), spec_version="2.1",
     )
 
     result = p.clean({"key": "identity--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=False)
@@ -464,7 +463,7 @@ def test_dictionary_property_ref_custom():
         p.clean({"key": "software--a2ac7670-f88f-424a-b3be-28f612f943f9"}, allow_custom=True)
 
     pfoo = DictionaryProperty(
-        valid_types=ReferenceProperty(valid_types=["SDO", "foo"]), spec_version="2.1"
+        valid_types=ReferenceProperty(valid_types=["SDO", "foo"]), spec_version="2.1",
     )
 
     with pytest.raises(CustomContentError):
@@ -476,7 +475,7 @@ def test_dictionary_property_ref_custom():
 
 def test_dictionary_property_values_strict_clean():
     prop = DictionaryProperty(
-        valid_types=[EnumProperty(["value1", "value2"]), IntegerProperty]
+        valid_types=[EnumProperty(["value1", "value2"]), IntegerProperty],
     )
 
     result = prop.clean({"key": "value1"}, allow_custom=False)

From 7df7b9ef60838ff32f0ef8f598e8a3eb78f1837a Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Mon, 11 Nov 2024 09:46:39 -0500
Subject: [PATCH 124/132] more on ARRAY

---
 .../database_backend_base.py                  |  13 +-
 .../database_backends/postgres_backend.py     |  16 +--
 .../datastore/relational_db/input_creation.py |  33 ++++-
 .../datastore/relational_db/relational_db.py  |   6 +-
 .../relational_db/relational_db_testing.py    |  59 ++++----
 .../datastore/relational_db/table_creation.py | 128 ++++++++++--------
 stix2/datastore/relational_db/utils.py        |   2 +-
 7 files changed, 144 insertions(+), 113 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index a3e74240..402eeb10 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -1,13 +1,11 @@
-
 from typing import Any
-import os
 
-from sqlalchemy import create_engine
-from sqlalchemy_utils import create_database, database_exists, drop_database
 from sqlalchemy import (  # create_engine,; insert,
     ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text, UniqueConstraint,
+    Integer, LargeBinary, Table, Text, UniqueConstraint, create_engine,
 )
+from sqlalchemy_utils import create_database, database_exists, drop_database
+
 
 class DatabaseBackend:
     def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
@@ -26,7 +24,7 @@ def _create_schemas(self):
         pass
 
     @staticmethod
-    def _determine_schema_name(stix_object):
+    def determine_schema_name(stix_object):
         return ""
 
     def _create_database(self):
@@ -81,6 +79,3 @@ def determine_sql_type_for_key_as_id():  # noqa: F811
     @staticmethod
     def array_allowed():
         return False
-
-
-
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index f88dbb10..ca8b0038 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -1,17 +1,17 @@
 import os
 from typing import Any
-from sqlalchemy.schema import CreateSchema
+
 from sqlalchemy import (  # create_engine,; insert,
     ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
     Integer, LargeBinary, Table, Text, UniqueConstraint,
 )
-
-from stix2.datastore.relational_db.utils import schema_for
+from sqlalchemy.schema import CreateSchema
 
 from stix2.base import (
-    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
+    _STIXBase,
 )
-
+from stix2.datastore.relational_db.utils import schema_for
 from stix2.properties import (
     BinaryProperty, BooleanProperty, DictionaryProperty,
     EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
@@ -22,6 +22,7 @@
 
 from .database_backend_base import DatabaseBackend
 
+
 class PostgresBackend(DatabaseBackend):
     default_database_connection_url = \
         f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:" + \
@@ -40,7 +41,7 @@ def _create_schemas(self):
             trans.execute(CreateSchema("sro", if_not_exists=True))
 
     @staticmethod
-    def _determine_schema_name(stix_object):
+    def determine_schema_name(stix_object):
         if isinstance(stix_object, _DomainObject):
             return "sdo"
         elif isinstance(stix_object, _Observable):
@@ -73,6 +74,3 @@ def determine_sql_type_for_timestamp_property():  # noqa: F811
     @staticmethod
     def array_allowed():
         return True
-
-
-
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index aa7e98bf..b9154c82 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -30,16 +30,17 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
     return {name: stix_object[name]}
 
 
-
 def instance_in_valid_types(cls, valid_types):
     for v in valid_types:
         if isinstance(v, cls):
             return True
     return False
 
+
 def is_valid_type(cls, valid_types):
     return cls in valid_types or instance_in_valid_types(cls, valid_types)
 
+
 @add_method(DictionaryProperty)
 def generate_insert_information(self, dictionary_name, stix_object, **kwargs):  # noqa: F811
     bindings = dict()
@@ -311,7 +312,8 @@ def generate_insert_for_external_references(data_sink, stix_object):
     return insert_statements
 
 
-def generate_insert_for_granular_markings(granular_markings_table, stix_object):
+def generate_insert_for_granular_markings(data_sink, granular_markings_table, stix_object):
+    db_backend = data_sink.db_backend
     insert_statements = list()
     granular_markings = stix_object["granular_markings"]
     for idx, granular_marking in enumerate(granular_markings):
@@ -322,7 +324,18 @@ def generate_insert_for_granular_markings(granular_markings_table, stix_object):
         marking_ref_value = granular_marking.get("marking_ref")
         if marking_ref_value:
             bindings["marking_ref"] = marking_ref_value
-        bindings["selectors"] = granular_marking.get("selectors")
+        if db_backend.array_allowed():
+            bindings["selectors"] = granular_marking.get("selectors")
+        else:
+            table = data_sink.tables_dictionary[
+                canonicalize_table_name(
+                    granular_markings_table + "_selector",
+                    db_backend.schema_for_core(),
+                )
+            ]
+            for sel in granular_marking.get("selectors"):
+                selector_bindings = {"id": stix_object["id"], "selector": sel}
+                insert_statements.append(insert(table).values(selector_bindings))
         insert_statements.append(insert(granular_markings_table).values(bindings))
     return insert_statements
 
@@ -359,12 +372,13 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
 
     core_insert_statement = insert(core_table).values(core_bindings)
     insert_statements.append(core_insert_statement)
+    object_marking_table_name = canonicalize_table_name("object_marking_refs", data_sink.db_backend.schema_for_core())
 
     if "object_marking_refs" in stix_object:
         if schema_name != "sco":
-            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sdo"]
+            object_markings_ref_table = data_sink.tables_dictionary[object_marking_table_name + "_sdo"]
         else:
-            object_markings_ref_table = data_sink.tables_dictionary["common.object_marking_refs_sco"]
+            object_markings_ref_table = data_sink.tables_dictionary[object_marking_table_name + "_sco"]
         insert_statements.extend(
             generate_insert_for_array_in_table(
                 object_markings_ref_table,
@@ -375,11 +389,16 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
 
     # Granular markings
     if "granular_markings" in stix_object:
+        granular_marking_table_name = canonicalize_table_name(
+            "granular_marking",
+            data_sink.db_backend.schema_for_core(),
+        )
         if schema_name != "sco":
-            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sdo"]
+            granular_marking_table = data_sink.tables_dictionary[granular_marking_table_name + "_sdo"]
         else:
-            granular_marking_table = data_sink.tables_dictionary["common.granular_marking_sco"]
+            granular_marking_table = data_sink.tables_dictionary[granular_marking_table_name + "_sco"]
         granular_input_statements = generate_insert_for_granular_markings(
+            data_sink,
             granular_marking_table,
             stix_object,
         )
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 6fbb027d..8d5db8d8 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -15,8 +15,6 @@
     canonicalize_table_name, schema_for, table_name_for,
 )
 from stix2.parsing import parse
-import stix2.registry
-import stix2.utils
 
 
 def _add(store, stix_data, allow_custom=True, version="2.1"):
@@ -202,7 +200,7 @@ def add(self, stix_data, version=None):
     #         return "common"
 
     def insert_object(self, stix_object):
-        schema_name = self.db_backend._determine_schema_name(stix_object)
+        schema_name = self.db_backend.determine_schema_name(stix_object)
         with self.db_backend.database_connection.begin() as trans:
             statements = generate_insert_for_object(self, stix_object, schema_name)
             for stmt in statements:
@@ -259,7 +257,7 @@ def get(self, stix_id, version=None, _composite_filters=None):
             stix_obj = read_object(
                 stix_id,
                 self.metadata,
-                conn
+                conn,
             )
 
         return stix_obj
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 262cf59a..bf0a142b 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -128,14 +128,14 @@ def dictionary_test():
         defanged=True,
         ctime="1980-02-23T05:43:28.2678Z",
         extensions={
-          "raster-image-ext": {
-              "exif_tags": {
-                  "Make": "Nikon",
-                  "Model": "D7000",
-                  "XResolution": 4928,
-                  "YResolution": 3264,
-              },
-          },
+            "raster-image-ext": {
+                "exif_tags": {
+                    "Make": "Nikon",
+                    "Model": "D7000",
+                    "XResolution": 4928,
+                    "YResolution": 3264,
+                },
+            },
         },
     )
 
@@ -148,34 +148,35 @@ def kill_chain_test():
         modified="2016-05-12T08:17:27.000Z",
         name="Spear Phishing",
         kill_chain_phases=[
-             {
-                 "kill_chain_name": "lockheed-martin-cyber-kill-chain",
-                 "phase_name": "reconnaissance",
-             },
+            {
+                "kill_chain_name": "lockheed-martin-cyber-kill-chain",
+                "phase_name": "reconnaissance",
+            },
         ],
         external_references=[
-             {
-                 "source_name": "capec",
-                 "external_id": "CAPEC-163",
-             },
+            {
+                "source_name": "capec",
+                "external_id": "CAPEC-163",
+            },
         ],
         granular_markings=[
-             {
-                 "lang": "en_US",
-                 "selectors": ["kill_chain_phases"],
-             },
-             {
-                 "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
-                 "selectors": ["external_references"],
-             },
+            {
+                "lang": "en_US",
+                "selectors": ["kill_chain_phases"],
+            },
+            {
+                "marking_ref": "marking-definition--50902d70-37ae-4f85-af68-3f4095493b42",
+                "selectors": ["external_references"],
+            },
         ], )
 
+
 @stix2.CustomObject(
     'x-custom-type',
-        properties=[
-            ("phases", stix2.properties.ListProperty(stix2.KillChainPhase)),
-            ("something_else", stix2.properties.IntegerProperty()),
-        ],
+    properties=[
+        ("phases", stix2.properties.ListProperty(stix2.KillChainPhase)),
+        ("something_else", stix2.properties.IntegerProperty()),
+    ],
 )
 class CustomClass:
     pass
@@ -275,8 +276,6 @@ def main():
 
         # store.add(tb)
 
-
-
         co = custom_obj()
 
         store.add(co)
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 1df146c4..e3f42ae4 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -19,7 +19,7 @@
     TimestampProperty, TypeProperty,
 )
 from stix2.v21.base import _Extension
-from stix2.v21.common import KillChainPhase, GranularMarking
+from stix2.v21.common import KillChainPhase
 
 
 def aux_table_property(prop, name, core_properties):
@@ -33,31 +33,35 @@ def aux_table_property(prop, name, core_properties):
 
 
 def create_array_column(property_name, contained_sql_type):
-    return Column(property_name,
-                  ARRAY(contained_sql_type),
-                  CheckConstraint(f"array_length({property_name}, 1) IS NOT NULL"),
-                  nullable=False)
+    return Column(
+        property_name,
+        ARRAY(contained_sql_type),
+        CheckConstraint(f"array_length({property_name}, 1) IS NOT NULL"),
+        nullable=False,
+    )
 
 
 def create_array_child_table(metadata, db_backend, table_name, property_name, contained_sql_type):
-        schema_name = determine_sql_type_from_stix(GranularMarking, db_backend)
-        columns = [
-            Column(
-                "id",
-                db_backend.determine_sql_type_for_key_as_id(),
-                ForeignKey(
-                    canonicalize_table_name(table_name, schema_name) + ".id",
-                    ondelete="CASCADE",
-                ),
-                nullable=False,
+    schema_name = db_backend.schema_for_core()
+    columns = [
+        Column(
+            "id",
+            db_backend.determine_sql_type_for_key_as_id(),
+            ForeignKey(
+                canonicalize_table_name(table_name, schema_name) + ".id",
+                ondelete="CASCADE",
             ),
-            Column(
-                property_name,
-                contained_sql_type,
-                nullable=False,
-            )
-        ]
-        return Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name)
+            nullable=False,
+            primary_key=True,
+        ),
+        Column(
+            property_name,
+            contained_sql_type,
+            nullable=False,
+        ),
+    ]
+    return Table(canonicalize_table_name(table_name + "_" + "selector", schema_name), metadata, *columns)
+
 
 def derive_column_name(prop):
     contained_property = prop.contained
@@ -167,13 +171,13 @@ def create_kill_chain_phases_table(name, metadata, db_backend, schema_name, tabl
 
 
 def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
-    tables = list()
     columns = [
         Column(
             "id",
             db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
             nullable=False,
+            primary_key=True,
         ),
         Column("lang", db_backend.determine_sql_type_for_string_property()),
         Column(
@@ -183,27 +187,36 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                 "marking_ref ~ '^marking-definition--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",
                 # noqa: E131
             ),
-        )
+        ),
     ]
     if db_backend.array_allowed():
+        child_table = False
         columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property()))
     else:
-        tables.append(create_array_child_table(metadata,
-                                               db_backend,
-                                               "granular_marking_" + sco_or_sdo,
-                                               "selector",
-                                               db_backend.determine_sql_type_for_string_property()))
-
-    tables.append(Table(
-                    "granular_marking_" + sco_or_sdo,
-                    metadata,
-                    *columns,
-                    CheckConstraint(
-                        """(lang IS NULL AND marking_ref IS NOT NULL)
-                           OR
-                           (lang IS NOT NULL AND marking_ref IS NULL)""",
-                    ),
-                    schema="common"))
+        child_table = True
+
+    tables = [
+        Table(
+            canonicalize_table_name("granular_marking_" + sco_or_sdo, db_backend.schema_for_core()),
+            metadata,
+            *columns,
+            CheckConstraint(
+            """(lang IS NULL AND marking_ref IS NOT NULL)
+                  OR
+                  (lang IS NOT NULL AND marking_ref IS NULL)""",
+            ),
+        ),
+    ]
+    if child_table:
+        tables.append(
+            create_array_child_table(
+                metadata,
+                db_backend,
+                "granular_marking_" + sco_or_sdo,
+                "selector",
+                db_backend.determine_sql_type_for_string_property(),
+            ),
+        )
     return tables
 
 
@@ -265,10 +278,12 @@ def create_core_table(metadata, db_backend, schema_name):
         columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
 
     tables = [
-        Table("core_" + schema_name,
-              metadata,
-              *columns,
-              schema=db_backend.schema_for_core())
+        Table(
+            "core_" + schema_name,
+            metadata,
+            *columns,
+            schema=db_backend.schema_for_core(),
+        ),
     ]
     return tables
 
@@ -565,7 +580,7 @@ def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
 def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     is_extension = kwargs.get('is_extension')
     tables = list()
-    # handle more complext embedded object before deciding if the ARRAY type is usable
+    # handle more complex embedded object before deciding if the ARRAY type is usable
     if isinstance(self.contained, EmbeddedObjectProperty):
         columns = list()
         columns.append(
@@ -636,6 +651,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
         tables.append(create_kill_chain_phases_table(name, metadata, db_backend, schema_name, table_name))
         return tables
     else:
+        # if ARRAY is not allowed, it is handled by a previous if clause
         if isinstance(self.contained, Property):
             sql_type = self.contained.determine_sql_type(db_backend)
             if sql_type:
@@ -824,15 +840,21 @@ def generate_object_table(
     return tables
 
 
+def add_tables(new_tables, tables):
+    if isinstance(new_tables, list):
+        tables.extend(new_tables)
+    else:
+        tables.append(new_tables)
+
+
 def create_core_tables(metadata, db_backend):
-    tables = [
-        create_core_table(metadata, db_backend, "sdo"),
-        create_granular_markings_table(metadata, db_backend, "sdo"),
-        create_core_table(metadata, db_backend, "sco"),
-        create_granular_markings_table(metadata, db_backend, "sco"),
-        create_object_markings_refs_table(metadata, db_backend, "sdo"),
-        create_object_markings_refs_table(metadata, db_backend, "sco"),
-    ]
+    tables = list()
+    add_tables(create_core_table(metadata, db_backend, "sdo"), tables)
+    add_tables(create_granular_markings_table(metadata, db_backend, "sdo"), tables)
+    add_tables(create_core_table(metadata, db_backend, "sco"), tables)
+    add_tables(create_granular_markings_table(metadata, db_backend, "sco"), tables)
+    add_tables(create_object_markings_refs_table(metadata, db_backend, "sdo"), tables)
+    add_tables(create_object_markings_refs_table(metadata, db_backend, "sco"), tables)
     tables.extend(create_external_references_tables(metadata, db_backend))
     return tables
 
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 76a4268b..d772172d 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -52,7 +52,7 @@ def canonicalize_table_name(table_name, schema_name=None):
     return inflection.underscore(full_name)
 
 
-_IGNORE_OBJECTS = [ "language-content"]
+_IGNORE_OBJECTS = ["language-content"]
 
 
 def get_all_subclasses(cls):

From cf9c0c9adee28e07066d951e620fd77bd4073915 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 13 Nov 2024 10:01:20 -0500
Subject: [PATCH 125/132] consistently make tables with schema argument, add
 sql-type pass methods

---
 .../database_backends/database_backend_base.py     | 14 ++++++++++++++
 stix2/datastore/relational_db/table_creation.py    | 12 +++++++-----
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index 402eeb10..07708ceb 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -40,10 +40,24 @@ def schema_for(stix_class):
     def schema_for_core():
         return ""
 
+    # you must implement the next 4 methods in the subclass
+
     @staticmethod
     def determine_sql_type_for_property():  # noqa: F811
         pass
 
+    @staticmethod
+    def determine_sql_type_for_binary_property():  # noqa: F811
+        pass
+
+    @staticmethod
+    def determine_sql_type_for_hex_property():  # noqa: F811
+        pass
+
+    @staticmethod
+    def determine_sql_type_for_timestamp_property():  # noqa: F811
+        pass
+
     @staticmethod
     def determine_sql_type_for_kill_chain_phase():  # noqa: F811
         return None
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index e3f42ae4..dea6fc86 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -1,8 +1,8 @@
 # from collections import OrderedDict
 
 from sqlalchemy import (  # create_engine,; insert,
-    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text, UniqueConstraint,
+    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, ForeignKey,
+    Integer, Table, Text, UniqueConstraint
 )
 
 from stix2.datastore.relational_db.add_method import add_method
@@ -60,7 +60,7 @@ def create_array_child_table(metadata, db_backend, table_name, property_name, co
             nullable=False,
         ),
     ]
-    return Table(canonicalize_table_name(table_name + "_" + "selector", schema_name), metadata, *columns)
+    return Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name)
 
 
 def derive_column_name(prop):
@@ -171,11 +171,12 @@ def create_kill_chain_phases_table(name, metadata, db_backend, schema_name, tabl
 
 
 def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
+    schema_name = db_backend.schema_for_core()
     columns = [
         Column(
             "id",
             db_backend.determine_sql_type_for_key_as_id(),
-            ForeignKey("common.core_" + sco_or_sdo + ".id", ondelete="CASCADE"),
+            ForeignKey(canonicalize_table_name("core_" + sco_or_sdo, schema_name) + ".id", ondelete="CASCADE"),
             nullable=False,
             primary_key=True,
         ),
@@ -197,7 +198,7 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
 
     tables = [
         Table(
-            canonicalize_table_name("granular_marking_" + sco_or_sdo, db_backend.schema_for_core()),
+            canonicalize_table_name("granular_marking_" + sco_or_sdo),
             metadata,
             *columns,
             CheckConstraint(
@@ -205,6 +206,7 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                   OR
                   (lang IS NOT NULL AND marking_ref IS NULL)""",
             ),
+            schema=schema_name
         ),
     ]
     if child_table:

From 4a019aab7b64dede286c927053a79f2ad3f04e4f Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Fri, 15 Nov 2024 16:43:17 -0500
Subject: [PATCH 126/132] more ARRAY

---
 .../database_backends/postgres_backend.py     |  2 +-
 .../datastore/relational_db/input_creation.py | 37 ++++++--
 .../datastore/relational_db/relational_db.py  | 51 ++--------
 .../relational_db/relational_db_testing.py    | 11 ++-
 .../datastore/relational_db/table_creation.py | 94 ++++++++++++-------
 stix2/datastore/relational_db/utils.py        |  2 +-
 6 files changed, 107 insertions(+), 90 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index ca8b0038..221e817e 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -31,7 +31,7 @@ class PostgresBackend(DatabaseBackend):
         f"{os.getenv('POSTGRES_PORT', '5432')}/postgres"
 
     def __init__(self, database_connection_url=default_database_connection_url, force_recreate=False, **kwargs: Any):
-        super().__init__(database_connection_url, force_recreate=False, **kwargs)
+        super().__init__(database_connection_url, force_recreate=force_recreate, **kwargs)
 
     def _create_schemas(self):
         with self.database_connection.begin() as trans:
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index b9154c82..8ed7e269 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -184,10 +184,10 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 @add_method(ListProperty)
 def generate_insert_information(   # noqa: F811
-    self, name, stix_object, level=0, is_extension=False,
+    self, name, stix_object, data_sink=None, level=0, is_extension=False,
     foreign_key_value=None, schema_name=None, **kwargs,
 ):
-    data_sink = kwargs.get("data_sink")
+    db_backend = data_sink.db_backend
     table_name = kwargs.get("table_name")
     if isinstance(self.contained, ReferenceProperty):
         insert_statements = list()
@@ -226,7 +226,7 @@ def generate_insert_information(   # noqa: F811
     elif isinstance(self.contained, EmbeddedObjectProperty):
         insert_statements = list()
         for value in stix_object[name]:
-            with data_sink.database_connection.begin() as trans:
+            with db_backend.database_connection.begin() as trans:
                 next_id = trans.execute(data_sink.sequence)
             table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
             bindings = {
@@ -248,7 +248,22 @@ def generate_insert_information(   # noqa: F811
             )
         return insert_statements
     else:
-        return {name: stix_object[name]}
+        if db_backend.array_allowed():
+            return {name: stix_object[name]}
+        else:
+            insert_statements = list()
+            table = data_sink.tables_dictionary[
+                canonicalize_table_name(
+                    table_name + "_" + name,
+                    schema_name,
+                )
+            ]
+            for elem in stix_object[name]:
+                bindings = {"id": stix_object["id"], name: elem}
+                insert_statements.append(insert(table).values(bindings))
+            return insert_statements
+
+
 
 
 @add_method(ReferenceProperty)
@@ -286,6 +301,7 @@ def generate_insert_for_array_in_table(table, values, foreign_key_value):
 
 
 def generate_insert_for_external_references(data_sink, stix_object):
+    db_backend = data_sink.db_backend
     insert_statements = list()
     next_id = None
     object_table = data_sink.tables_dictionary["common.external_references"]
@@ -295,7 +311,7 @@ def generate_insert_for_external_references(data_sink, stix_object):
             if prop in er:
                 bindings[prop] = er[prop]
         if "hashes" in er:
-            with data_sink.database_connection.begin() as trans:
+            with db_backend.database_connection.begin() as trans:
                 next_id = trans.execute(data_sink.sequence)
             bindings["hash_ref_id"] = next_id
         else:
@@ -326,17 +342,22 @@ def generate_insert_for_granular_markings(data_sink, granular_markings_table, st
             bindings["marking_ref"] = marking_ref_value
         if db_backend.array_allowed():
             bindings["selectors"] = granular_marking.get("selectors")
+            insert_statements.append(insert(granular_markings_table).values(bindings))
         else:
+            with db_backend.database_connection.begin() as trans:
+                next_id = trans.execute(data_sink.sequence)
+            bindings["selectors"] = next_id
+            insert_statements.append(insert(granular_markings_table).values(bindings))
             table = data_sink.tables_dictionary[
                 canonicalize_table_name(
-                    granular_markings_table + "_selector",
+                    granular_markings_table.name + "_selector",
                     db_backend.schema_for_core(),
                 )
             ]
             for sel in granular_marking.get("selectors"):
-                selector_bindings = {"id": stix_object["id"], "selector": sel}
+                selector_bindings = {"id": next_id, "selector": sel}
                 insert_statements.append(insert(table).values(selector_bindings))
-        insert_statements.append(insert(granular_markings_table).values(bindings))
+
     return insert_statements
 
 
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 8d5db8d8..6fc1fe8a 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -57,7 +57,7 @@ def _add(store, stix_data, allow_custom=True, version="2.1"):
 class RelationalDBStore(DataStoreMixin):
     def __init__(
         self, db_backend, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes,
+        instantiate_database=True, print_sql=False, *stix_object_classes,
     ):
         """
         Initialize this store.
@@ -91,6 +91,7 @@ def __init__(
             ),
             sink=RelationalDBSink(
                 db_backend,
+                print_sql=print_sql,
                 allow_custom=allow_custom,
                 version=version,
                 instantiate_database=instantiate_database,
@@ -102,7 +103,7 @@ def __init__(
 class RelationalDBSink(DataSink):
     def __init__(
         self, db_backend, allow_custom=True, version=None,
-        instantiate_database=True, *stix_object_classes, metadata=None,
+        instantiate_database=True, print_sql=False, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this sink.  Only one of stix_object_classes and metadata
@@ -134,15 +135,6 @@ def __init__(
 
         self.db_backend = db_backend
 
-        # if isinstance(database_connection_or_url, str):
-        #     self.database_connection = create_engine(database_connection_or_url)
-        # else:
-        #     self.database_connection = database_connection_or_url
-
-        # self.database_exists = database_exists(self.database_connection.url)
-        # if force_recreate:
-        #     self._create_database()
-
         if metadata:
             self.metadata = metadata
         else:
@@ -161,44 +153,21 @@ def __init__(
         if instantiate_database:
             if not self.db_backend.database_exists:
                 self.db_backend._create_database()
+            # else:
+            #     self.clear_tables()
             self.db_backend._create_schemas()
-            self._instantiate_database()
+            self._instantiate_database(print_sql)
 
-    # def _create_schemas(self):
-    #     with self.database_connection.begin() as trans:
-    #         trans.execute(CreateSchema("common", if_not_exists=True))
-    #         trans.execute(CreateSchema("sdo", if_not_exists=True))
-    #         trans.execute(CreateSchema("sco", if_not_exists=True))
-    #         trans.execute(CreateSchema("sro", if_not_exists=True))
-
-    def _instantiate_database(self):
+    def _instantiate_database(self, print_sql=False):
         self.metadata.create_all(self.db_backend.database_connection)
-
-    # def _create_database(self):
-    #     if self.database_exists:
-    #         drop_database(self.database_connection.url)
-    #     create_database(self.database_connection.url)
-    #     self.database_exists = database_exists(self.database_connection.url)
-
-    def generate_stix_schema(self):
-        for t in self.metadata.tables.values():
-            print(CreateTable(t).compile(self.db_backend.database_connection))
+        if print_sql:
+            for t in self.metadata.tables.values():
+                print(CreateTable(t).compile(self.db_backend.database_connection))
 
     def add(self, stix_data, version=None):
         _add(self, stix_data)
     add.__doc__ = _add.__doc__
 
-    # @staticmethod
-    # def _determine_schema_name(stix_object):
-    #     if isinstance(stix_object, _DomainObject):
-    #         return "sdo"
-    #     elif isinstance(stix_object, _Observable):
-    #         return "sco"
-    #     elif isinstance(stix_object, _RelationshipObject):
-    #         return "sro"
-    #     elif isinstance(stix_object, _MetaObject):
-    #         return "common"
-
     def insert_object(self, stix_object):
         schema_name = self.db_backend.determine_schema_name(stix_object)
         with self.db_backend.database_connection.begin() as trans:
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index bf0a142b..454a69cd 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -254,15 +254,16 @@ def custom_obj():
 
 def main():
     store = RelationalDBStore(
-        PostgresBackend("postgresql://localhost/stix-data-sink"),
+        PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
         False,
         None,
         True,
+        print_sql=True,
     )
 
     if store.sink.db_backend.database_exists:
-        store.sink.generate_stix_schema()
-        store.sink.clear_tables()
+        # store.sink.generate_stix_schema()
+        # store.sink.clear_tables()
 
         # td = test_dictionary()
 
@@ -295,8 +296,8 @@ def main():
         dict_example = dictionary_test()
         store.add(dict_example)
 
-        read_obj = store.get(directory_stix_object.id)
-        print(read_obj)
+        # read_obj = store.get(directory_stix_object.id)
+        # print(read_obj)
     else:
         print("database does not exist")
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index dea6fc86..1a1a94df 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -32,12 +32,12 @@ def aux_table_property(prop, name, core_properties):
         return False
 
 
-def create_array_column(property_name, contained_sql_type):
+def create_array_column(property_name, contained_sql_type, optional):
     return Column(
         property_name,
         ARRAY(contained_sql_type),
-        CheckConstraint(f"array_length({property_name}, 1) IS NOT NULL"),
-        nullable=False,
+        CheckConstraint(f"{property_name} IS NULL or array_length({property_name}, 1) IS NOT NULL"),
+        nullable=optional
     )
 
 
@@ -47,12 +47,12 @@ def create_array_child_table(metadata, db_backend, table_name, property_name, co
         Column(
             "id",
             db_backend.determine_sql_type_for_key_as_id(),
-            ForeignKey(
-                canonicalize_table_name(table_name, schema_name) + ".id",
-                ondelete="CASCADE",
-            ),
+            # ForeignKey(
+            #     canonicalize_table_name(table_name, schema_name) + ".id",
+            #     ondelete="CASCADE",
+            # ),
             nullable=False,
-            primary_key=True,
+            #primary_key=True,
         ),
         Column(
             property_name,
@@ -178,7 +178,7 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
             db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(canonicalize_table_name("core_" + sco_or_sdo, schema_name) + ".id", ondelete="CASCADE"),
             nullable=False,
-            primary_key=True,
+            # primary_key=not db_backend.array_allowed(),
         ),
         Column("lang", db_backend.determine_sql_type_for_string_property()),
         Column(
@@ -191,35 +191,61 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
         ),
     ]
     if db_backend.array_allowed():
-        child_table = False
-        columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property()))
-    else:
-        child_table = True
-
-    tables = [
-        Table(
-            canonicalize_table_name("granular_marking_" + sco_or_sdo),
-            metadata,
-            *columns,
-            CheckConstraint(
-            """(lang IS NULL AND marking_ref IS NOT NULL)
-                  OR
-                  (lang IS NOT NULL AND marking_ref IS NULL)""",
-            ),
-            schema=schema_name
-        ),
-    ]
-    if child_table:
-        tables.append(
-            create_array_child_table(
+        columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property(), False))
+        return [
+            Table(
+                "granular_marking_" + sco_or_sdo,
                 metadata,
-                db_backend,
+                *columns,
+                CheckConstraint(
+                """(lang IS NULL AND marking_ref IS NOT NULL)
+                      OR
+                      (lang IS NOT NULL AND marking_ref IS NULL)""",
+                ),
+                schema=schema_name
+            ),
+        ]
+    else:
+        columns.append(Column(
+                "selectors",
+                db_backend.determine_sql_type_for_key_as_int(),
+                unique=True
+            )
+        )
+        tables = [
+            Table(
                 "granular_marking_" + sco_or_sdo,
+                metadata,
+                *columns,
+                CheckConstraint(
+                    """(lang IS NULL AND marking_ref IS NOT NULL)
+                          OR
+                          (lang IS NOT NULL AND marking_ref IS NULL)""",
+                ),
+                schema=schema_name
+            ),
+        ]
+        schema_name = db_backend.schema_for_core()
+        columns = [
+            Column(
+                "id",
+                db_backend.determine_sql_type_for_key_as_int(),
+                ForeignKey(
+                    canonicalize_table_name("granular_marking_" + sco_or_sdo, schema_name) + ".selectors",
+                    ondelete="CASCADE",
+                ),
+                # primary_key=True,
+                nullable=False,
+            ),
+            Column(
                 "selector",
                 db_backend.determine_sql_type_for_string_property(),
+                nullable=False,
             ),
-        )
-    return tables
+        ]
+        tables.append(Table(canonicalize_table_name("granular_marking_" + sco_or_sdo + "_" + "selector"),
+                            metadata, *columns, schema=schema_name))
+        return tables
 
 
 def create_external_references_tables(metadata, db_backend):
@@ -275,7 +301,7 @@ def create_core_table(metadata, db_backend, schema_name):
         ]
         columns.extend(sdo_columns)
         if db_backend.array_allowed():
-            columns.append(create_array_column("labels", db_backend.determine_sql_type_for_string_property())),
+            columns.append(create_array_column("labels", db_backend.determine_sql_type_for_string_property(), True))
     else:
         columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
 
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index d772172d..92806b38 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -148,7 +148,7 @@ def determine_sql_type_from_stix(cls_or_inst, db_backend):  # noqa: F811
     elif is_class_or_instance(cls_or_inst, IntegerProperty):
         return db_backend.determine_sql_type_for_integer_property()
     elif is_class_or_instance(cls_or_inst, StringProperty):
-        return db_backend.determine_sql_type_for_integer_property()
+        return db_backend.determine_sql_type_for_string_property()
     elif is_class_or_instance(cls_or_inst, ReferenceProperty):
         db_backend.determine_sql_type_for_reference_property()
     elif is_class_or_instance(cls_or_inst, TimestampProperty):

From 35ce57acbbd85b9ebcc3a6f66c2d20d7d0391f1c Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 17 Nov 2024 09:45:06 -0500
Subject: [PATCH 127/132] next_id method, stix_type_name vs. schema_name

---
 .../database_backend_base.py                  |  20 ++-
 .../database_backends/postgres_backend.py     |   5 +-
 .../datastore/relational_db/input_creation.py |  61 +++++---
 .../datastore/relational_db/relational_db.py  |   8 +-
 .../relational_db/relational_db_testing.py    | 141 +++++++++---------
 .../datastore/relational_db/table_creation.py | 127 +++++++---------
 6 files changed, 186 insertions(+), 176 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index 07708ceb..118db629 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -1,11 +1,14 @@
 from typing import Any
 
-from sqlalchemy import (  # create_engine,; insert,
-    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text, UniqueConstraint, create_engine,
+from sqlalchemy import (
+    create_engine, Boolean, Float, Integer, LargeBinary, Text, TIMESTAMP,
 )
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
+from stix2.base import (
+    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
+    _STIXBase,
+)
 
 class DatabaseBackend:
     def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
@@ -27,6 +30,17 @@ def _create_schemas(self):
     def determine_schema_name(stix_object):
         return ""
 
+    @staticmethod
+    def determine_stix_type(stix_object):
+        if isinstance(stix_object, _DomainObject):
+            return "sdo"
+        elif isinstance(stix_object, _Observable):
+            return "sco"
+        elif isinstance(stix_object, _RelationshipObject):
+            return "sro"
+        elif isinstance(stix_object, _MetaObject):
+            return "common"
+
     def _create_database(self):
         if self.database_exists:
             drop_database(self.database_connection.url)
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index 221e817e..e80fd1bd 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -1,9 +1,8 @@
 import os
 from typing import Any
 
-from sqlalchemy import (  # create_engine,; insert,
-    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, Float, ForeignKey,
-    Integer, LargeBinary, Table, Text, UniqueConstraint,
+from sqlalchemy import (
+    TIMESTAMP, LargeBinary, Text,
 )
 from sqlalchemy.schema import CreateSchema
 
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index 8ed7e269..ff308517 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -226,8 +226,7 @@ def generate_insert_information(   # noqa: F811
     elif isinstance(self.contained, EmbeddedObjectProperty):
         insert_statements = list()
         for value in stix_object[name]:
-            with db_backend.database_connection.begin() as trans:
-                next_id = trans.execute(data_sink.sequence)
+            next_id = data_sink.next_id()
             table = data_sink.tables_dictionary[canonicalize_table_name(table_name + "_" + name, schema_name)]
             bindings = {
                 "id": foreign_key_value,
@@ -289,12 +288,12 @@ def derive_column_name(prop):
         return "value"
 
 
-def generate_insert_for_array_in_table(table, values, foreign_key_value):
+def generate_insert_for_array_in_table(table, values, foreign_key_value, column_name="ref_id"):
     insert_statements = list()
     for idx, item in enumerate(values):
         bindings = {
             "id": foreign_key_value,
-            "ref_id": item,
+            column_name: item,
         }
         insert_statements.append(insert(table).values(bindings))
     return insert_statements
@@ -311,12 +310,11 @@ def generate_insert_for_external_references(data_sink, stix_object):
             if prop in er:
                 bindings[prop] = er[prop]
         if "hashes" in er:
-            with db_backend.database_connection.begin() as trans:
-                next_id = trans.execute(data_sink.sequence)
+            next_id = data_sink.next_id()
             bindings["hash_ref_id"] = next_id
-        else:
-            # hash_ref_id is non-NULL, so -1 means there are no hashes
-            bindings["hash_ref_id"] = -1
+        # else:
+        #     # hash_ref_id is non-NULL, so -1 means there are no hashes
+        #     bindings["hash_ref_id"] = -1
         er_insert_statement = insert(object_table).values(bindings)
         insert_statements.append(er_insert_statement)
 
@@ -344,8 +342,7 @@ def generate_insert_for_granular_markings(data_sink, granular_markings_table, st
             bindings["selectors"] = granular_marking.get("selectors")
             insert_statements.append(insert(granular_markings_table).values(bindings))
         else:
-            with db_backend.database_connection.begin() as trans:
-                next_id = trans.execute(data_sink.sequence)
+            next_id = data_sink.next_id()
             bindings["selectors"] = next_id
             insert_statements.append(insert(granular_markings_table).values(bindings))
             table = data_sink.tables_dictionary[
@@ -376,27 +373,43 @@ def generate_insert_for_granular_markings(data_sink, granular_markings_table, st
 #     return sql_bindings_tuples
 
 
-def generate_insert_for_core(data_sink, stix_object, core_properties, schema_name):
-    if schema_name in ["sdo", "sro", "common"]:
-        core_table = data_sink.tables_dictionary["common.core_sdo"]
+def generate_insert_for_core(data_sink, stix_object, core_properties, stix_type_name, schema_name):
+    db_backend = data_sink.db_backend
+    if stix_type_name in ["sdo", "sro", "common"]:
+        core_table = data_sink.tables_dictionary[db_backend.schema_for_core() + "." + "core_sdo"]
     else:
-        core_table = data_sink.tables_dictionary["common.core_sco"]
+        core_table = data_sink.tables_dictionary[db_backend.schema_for_core() + "." + "core_sco"]
     insert_statements = list()
     core_bindings = {}
 
-    for prop_name, value in stix_object.items():
+    child_table_properties = ["object_marking_refs", "granular_markings", "external_references", "type"]
+    if "labels" in core_properties and not db_backend.array_allowed():
+        child_table_properties.append("labels")
 
+    for prop_name, value in stix_object.items():
         if prop_name in core_properties:
             # stored in separate tables below, skip here
-            if prop_name not in {"object_marking_refs", "granular_markings", "external_references", "type"}:
+            if prop_name not in child_table_properties:
                 core_bindings[prop_name] = value
 
     core_insert_statement = insert(core_table).values(core_bindings)
     insert_statements.append(core_insert_statement)
-    object_marking_table_name = canonicalize_table_name("object_marking_refs", data_sink.db_backend.schema_for_core())
+
+    if "labels" in stix_object:
+        label_table_name = canonicalize_table_name(core_table.name + "_labels", data_sink.db_backend.schema_for_core())
+        labels_table = data_sink.tables_dictionary[label_table_name]
+        insert_statements.extend(
+            generate_insert_for_array_in_table(
+                labels_table,
+                stix_object["labels"],
+                stix_object["id"],
+                column_name="label"
+            ))
 
     if "object_marking_refs" in stix_object:
-        if schema_name != "sco":
+        object_marking_table_name = canonicalize_table_name("object_marking_refs",
+                                                            data_sink.db_backend.schema_for_core())
+        if stix_type_name != "sco":
             object_markings_ref_table = data_sink.tables_dictionary[object_marking_table_name + "_sdo"]
         else:
             object_markings_ref_table = data_sink.tables_dictionary[object_marking_table_name + "_sco"]
@@ -414,7 +427,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, schema_nam
             "granular_marking",
             data_sink.db_backend.schema_for_core(),
         )
-        if schema_name != "sco":
+        if stix_type_name != "sco":
             granular_marking_table = data_sink.tables_dictionary[granular_marking_table_name + "_sdo"]
         else:
             granular_marking_table = data_sink.tables_dictionary[granular_marking_table_name + "_sco"]
@@ -474,18 +487,18 @@ def generate_insert_for_sub_object(
     return insert_statements
 
 
-def generate_insert_for_object(data_sink, stix_object, schema_name, level=0):
+def generate_insert_for_object(data_sink, stix_object, stix_type_name, schema_name, level=0):
     insert_statements = list()
     bindings = dict()
-    if schema_name == "sco":
+    if stix_type_name == "sco":
         core_properties = SCO_COMMON_PROPERTIES
-    elif schema_name in ["sdo", "sro", "common"]:
+    elif stix_type_name in ["sdo", "sro", "common"]:
         core_properties = SDO_COMMON_PROPERTIES
     else:
         core_properties = list()
     type_name = stix_object["type"]
     if core_properties:
-        insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, schema_name))
+        insert_statements.extend(generate_insert_for_core(data_sink, stix_object, core_properties, stix_type_name, schema_name))
     if "id" in stix_object:
         foreign_key_value = stix_object["id"]
     else:
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 6fc1fe8a..56cd32d0 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -165,13 +165,14 @@ def _instantiate_database(self, print_sql=False):
                 print(CreateTable(t).compile(self.db_backend.database_connection))
 
     def add(self, stix_data, version=None):
-        _add(self, stix_data)
+        _add(self, stix_data, self.allow_custom)
     add.__doc__ = _add.__doc__
 
     def insert_object(self, stix_object):
         schema_name = self.db_backend.determine_schema_name(stix_object)
+        stix_type_name = self.db_backend.determine_stix_type(stix_object)
         with self.db_backend.database_connection.begin() as trans:
-            statements = generate_insert_for_object(self, stix_object, schema_name)
+            statements = generate_insert_for_object(self, stix_object, stix_type_name, schema_name)
             for stmt in statements:
                 print("executing: ", stmt)
                 trans.execute(stmt)
@@ -185,6 +186,9 @@ def clear_tables(self):
                 print(f'delete_stmt: {delete_stmt}')
                 trans.execute(delete_stmt)
 
+    def next_id(self):
+        with self.db_backend.database_connection.begin() as trans:
+            return trans.execute(self.sequence)
 
 class RelationalDBSource(DataSource):
     def __init__(
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 454a69cd..926803a2 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -52,18 +52,24 @@ def windows_registry_key_example():
 
 
 def malware_with_all_required_properties():
-    ref = stix2.v21.ExternalReference(
+    ref1 = stix2.v21.ExternalReference(
         source_name="veris",
         external_id="0001AA7F-C601-424A-B2B8-BE6C9F5164E7",
-        # hashes={
-        #    "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b",
-        # },
+        hashes={
+           "SHA-256": "6db12788c37247f2316052e142f42f4b259d6561751e5f401a1ae2a6df9c674b",
+           "MD5":  "3773a88f65a5e780c8dff9cdc3a056f3",
+        },
         url="https://github.com/vz-risk/VCDB/blob/master/data/json/0001AA7F-C601-424A-B2B8-BE6C9F5164E7.json",
     )
+    ref2 = stix2.v21.ExternalReference(
+        source_name="ACME Threat Intel",
+        description="Threat report",
+        url="http://www.example.com/threat-report.pdf"
+    )
     now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
 
     malware = stix2.v21.Malware(
-        external_references=[ref],
+        external_references=[ref1, ref2],
         type="malware",
         id="malware--9c4638ec-f1de-4ddb-abf4-1b760417654e",
         created=now,
@@ -195,61 +201,61 @@ def custom_obj():
     return obj
 
 
-# @stix2.CustomObject(
-#     "test-object", [
-#         ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
-#     ],
-#     "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
-#     is_sdo=True
-# )
-# class TestClass:
-#     pass
-#
-#
-# def test_binary_list():
-#     return TestClass(prop_name=["AREi", "7t3M"])
-#
-# @stix2.CustomObject(
-#         "test2-object", [
-#             ("prop_name", stix2.properties.ListProperty(
-#                 stix2.properties.HexProperty()
-#             ))
-#         ],
-#         "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
-#         is_sdo=True
-#     )
-#
-# class Test2Class:
-#         pass
-#
-# def test_hex_list():
-#     return Test2Class(
-#         prop_name=["1122", "fedc"]
-#     )
-#
-# @stix2.CustomObject(
-#         "test3-object", [
-#             ("prop_name",
-#                  stix2.properties.DictionaryProperty(
-#                      valid_types=[
-#                          stix2.properties.IntegerProperty,
-#                          stix2.properties.FloatProperty,
-#                          stix2.properties.StringProperty
-#                      ]
-#                  )
-#              )
-#         ],
-#         "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
-#         is_sdo=True
-#     )
-# class Test3Class:
-#     pass
-#
-#
-# def test_dictionary():
-#     return Test3Class(
-#         prop_name={"a": 1, "b": 2.3, "c": "foo"}
-#     )
+@stix2.CustomObject(
+    "test-object", [
+        ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
+    ],
+    "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
+    is_sdo=True
+)
+class TestClass:
+    pass
+
+
+def test_binary_list():
+    return TestClass(prop_name=["AREi", "7t3M"])
+
+@stix2.CustomObject(
+        "test2-object", [
+            ("prop_name", stix2.properties.ListProperty(
+                stix2.properties.HexProperty()
+            ))
+        ],
+        "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
+        is_sdo=True
+    )
+
+class Test2Class:
+        pass
+
+def test_hex_list():
+    return Test2Class(
+        prop_name=["1122", "fedc"]
+    )
+
+@stix2.CustomObject(
+        "test3-object", [
+            ("prop_name",
+                 stix2.properties.DictionaryProperty(
+                     valid_types=[
+                         stix2.properties.IntegerProperty,
+                         stix2.properties.FloatProperty,
+                         stix2.properties.StringProperty
+                     ]
+                 )
+             )
+        ],
+        "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
+        is_sdo=True
+    )
+class Test3Class:
+    pass
+
+
+def test_dictionary():
+    return Test3Class(
+        prop_name={"a": 1, "b": 2.3, "c": "foo"}
+    )
 
 
 def main():
@@ -262,19 +268,17 @@ def main():
     )
 
     if store.sink.db_backend.database_exists:
-        # store.sink.generate_stix_schema()
-        # store.sink.clear_tables()
 
         # td = test_dictionary()
-
+        #
         # store.add(td)
-
+        #
         # th = test_hex_list()
-
+        #
         # store.add(th)
-
+        #
         # tb = test_binary_list()
-
+        #
         # store.add(tb)
 
         co = custom_obj()
@@ -296,6 +300,9 @@ def main():
         dict_example = dictionary_test()
         store.add(dict_example)
 
+        malware = malware_with_all_required_properties()
+        store.add(malware)
+
         # read_obj = store.get(directory_stix_object.id)
         # print(read_obj)
     else:
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index 1a1a94df..d1feeddb 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -22,16 +22,6 @@
 from stix2.v21.common import KillChainPhase
 
 
-def aux_table_property(prop, name, core_properties):
-    if isinstance(prop, ListProperty) and name not in core_properties:
-        contained_property = prop.contained
-        return not isinstance(contained_property, (StringProperty, IntegerProperty, FloatProperty))
-    elif isinstance(prop, DictionaryProperty) and name not in core_properties:
-        return True
-    else:
-        return False
-
-
 def create_array_column(property_name, contained_sql_type, optional):
     return Column(
         property_name,
@@ -41,18 +31,17 @@ def create_array_column(property_name, contained_sql_type, optional):
     )
 
 
-def create_array_child_table(metadata, db_backend, table_name, property_name, contained_sql_type):
+def create_array_child_table(metadata, db_backend, parent_table_name, table_name_suffix, property_name, contained_sql_type):
     schema_name = db_backend.schema_for_core()
     columns = [
         Column(
             "id",
             db_backend.determine_sql_type_for_key_as_id(),
-            # ForeignKey(
-            #     canonicalize_table_name(table_name, schema_name) + ".id",
-            #     ondelete="CASCADE",
-            # ),
+            ForeignKey(
+                 canonicalize_table_name(parent_table_name, schema_name) + ".id",
+                 ondelete="CASCADE",
+            ),
             nullable=False,
-            #primary_key=True,
         ),
         Column(
             property_name,
@@ -60,7 +49,7 @@ def create_array_child_table(metadata, db_backend, table_name, property_name, co
             nullable=False,
         ),
     ]
-    return Table(canonicalize_table_name(table_name + "_" + "selector"), metadata, *columns, schema=schema_name)
+    return Table(parent_table_name + table_name_suffix, metadata, *columns, schema=schema_name)
 
 
 def derive_column_name(prop):
@@ -109,10 +98,10 @@ def create_hashes_table(name, metadata, db_backend, schema_name, table_name, key
         Column(
             "id",
             key_type,
-            ForeignKey(
-                canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
-                ondelete="CASCADE",
-            ),
+            # ForeignKey(
+            #     canonicalize_table_name(table_name, schema_name) + (".hash_ref_id" if table_name == "external_references" else ".id"),
+            #     ondelete="CASCADE",
+            # ),
 
             nullable=False,
         ),
@@ -172,13 +161,13 @@ def create_kill_chain_phases_table(name, metadata, db_backend, schema_name, tabl
 
 def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
     schema_name = db_backend.schema_for_core()
+    tables = list()
     columns = [
         Column(
             "id",
             db_backend.determine_sql_type_for_key_as_id(),
             ForeignKey(canonicalize_table_name("core_" + sco_or_sdo, schema_name) + ".id", ondelete="CASCADE"),
             nullable=False,
-            # primary_key=not db_backend.array_allowed(),
         ),
         Column("lang", db_backend.determine_sql_type_for_string_property()),
         Column(
@@ -192,19 +181,7 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
     ]
     if db_backend.array_allowed():
         columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property(), False))
-        return [
-            Table(
-                "granular_marking_" + sco_or_sdo,
-                metadata,
-                *columns,
-                CheckConstraint(
-                """(lang IS NULL AND marking_ref IS NOT NULL)
-                      OR
-                      (lang IS NOT NULL AND marking_ref IS NULL)""",
-                ),
-                schema=schema_name
-            ),
-        ]
+
     else:
         columns.append(Column(
                 "selectors",
@@ -212,21 +189,8 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                 unique=True
             )
         )
-        tables = [
-            Table(
-                "granular_marking_" + sco_or_sdo,
-                metadata,
-                *columns,
-                CheckConstraint(
-                    """(lang IS NULL AND marking_ref IS NOT NULL)
-                          OR
-                          (lang IS NOT NULL AND marking_ref IS NULL)""",
-                ),
-                schema=schema_name
-            ),
-        ]
-        schema_name = db_backend.schema_for_core()
-        columns = [
+
+        child_columns = [
             Column(
                 "id",
                 db_backend.determine_sql_type_for_key_as_int(),
@@ -234,7 +198,6 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                     canonicalize_table_name("granular_marking_" + sco_or_sdo, schema_name) + ".selectors",
                     ondelete="CASCADE",
                 ),
-                # primary_key=True,
                 nullable=False,
             ),
             Column(
@@ -244,8 +207,20 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
             ),
         ]
         tables.append(Table(canonicalize_table_name("granular_marking_" + sco_or_sdo + "_" + "selector"),
-                            metadata, *columns, schema=schema_name))
-        return tables
+                            metadata, *child_columns, schema=schema_name))
+    tables.append(
+        Table(
+            "granular_marking_" + sco_or_sdo,
+            metadata,
+            *columns,
+            CheckConstraint(
+                """(lang IS NULL AND marking_ref IS NOT NULL)
+                      OR
+                      (lang IS NOT NULL AND marking_ref IS NULL)""",
+            ),
+            schema=schema_name
+        ))
+    return tables
 
 
 def create_external_references_tables(metadata, db_backend):
@@ -263,7 +238,7 @@ def create_external_references_tables(metadata, db_backend):
         Column("url", db_backend.determine_sql_type_for_string_property()),
         Column("external_id", db_backend.determine_sql_type_for_string_property()),
         # all such keys are generated using the global sequence.
-        Column("hash_ref_id", db_backend.determine_sql_type_for_key_as_int(), primary_key=True, autoincrement=False),
+        Column("hash_ref_id", db_backend.determine_sql_type_for_key_as_int(), autoincrement=False),
     ]
     return [
         Table("external_references", metadata, *columns, schema="common"),
@@ -271,8 +246,9 @@ def create_external_references_tables(metadata, db_backend):
     ]
 
 
-def create_core_table(metadata, db_backend, schema_name):
+def create_core_table(metadata, db_backend, stix_type_name):
     tables = list()
+    table_name = "core_" + stix_type_name
     columns = [
         Column(
             "id",
@@ -284,7 +260,7 @@ def create_core_table(metadata, db_backend, schema_name):
         ),
         Column("spec_version", db_backend.determine_sql_type_for_string_property(), default="2.1"),
     ]
-    if schema_name == "sdo":
+    if stix_type_name == "sdo":
         sdo_columns = [
             Column(
                 "created_by_ref",
@@ -293,26 +269,32 @@ def create_core_table(metadata, db_backend, schema_name):
                     "created_by_ref ~ '^identity--[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$'",   # noqa: E131
                 ),
             ),
-            Column("created", TIMESTAMP(timezone=True)),
-            Column("modified", TIMESTAMP(timezone=True)),
-            Column("revoked", Boolean),
-            Column("confidence", Integer),
+            Column("created", db_backend.determine_sql_type_for_timestamp_property()),
+            Column("modified", db_backend.determine_sql_type_for_timestamp_property()),
+            Column("revoked", db_backend.determine_sql_type_for_boolean_property()),
+            Column("confidence", db_backend.determine_sql_type_for_integer_property()),
             Column("lang", db_backend.determine_sql_type_for_string_property()),
         ]
         columns.extend(sdo_columns)
         if db_backend.array_allowed():
             columns.append(create_array_column("labels", db_backend.determine_sql_type_for_string_property(), True))
+        else:
+            tables.append(create_array_child_table(metadata,
+                                                   db_backend,
+                                                   table_name,
+                                                   "_labels",
+                                                   "label",
+                                                   db_backend.determine_sql_type_for_string_property()))
     else:
         columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
 
-    tables = [
+    tables.append(
         Table(
-            "core_" + schema_name,
+            table_name,
             metadata,
             *columns,
             schema=db_backend.schema_for_core(),
-        ),
-    ]
+        ))
     return tables
 
 
@@ -440,12 +422,9 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
             else:
                 contained_class = self.valid_types[0].contained
                 columns.append(
-                    Column(
-                        "value",
-                        # its an instance, not a class
-                        ARRAY(contained_class.determine_sql_type(db_backend)),
-                        nullable=False,
-                    ),
+                    create_array_column("value",
+                                        contained_class.determine_sql_type(db_backend),
+                                        False)
                 )
         else:
             for column_type in self.valid_types:
@@ -681,13 +660,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
     else:
         # if ARRAY is not allowed, it is handled by a previous if clause
         if isinstance(self.contained, Property):
-            sql_type = self.contained.determine_sql_type(db_backend)
-            if sql_type:
-                return Column(
-                    name,
-                    ARRAY(sql_type),
-                    nullable=not (self.required),
-                )
+            return create_array_column(name, self.contained.determine_sql_type(db_backend), not self.required)
 
 
 def ref_column(name, specifics, db_backend, auth_type=0):

From 3276661e56a24d021c80da449f5ea69ba7181583 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Sun, 17 Nov 2024 14:18:22 -0500
Subject: [PATCH 128/132] handle hex_property in list

---
 stix2/datastore/__init__.py                   |  2 +-
 .../database_backend_base.py                  |  8 +-
 .../database_backends/postgres_backend.py     | 14 +---
 .../datastore/relational_db/input_creation.py | 27 ++++---
 .../datastore/relational_db/relational_db.py  | 19 +++--
 .../relational_db/relational_db_testing.py    | 75 ++++++++++---------
 .../datastore/relational_db/table_creation.py | 72 ++++++++++++------
 stix2/datastore/relational_db/utils.py        |  3 -
 8 files changed, 120 insertions(+), 100 deletions(-)

diff --git a/stix2/datastore/__init__.py b/stix2/datastore/__init__.py
index 39229805..715c6e6b 100644
--- a/stix2/datastore/__init__.py
+++ b/stix2/datastore/__init__.py
@@ -212,7 +212,7 @@ def add(self, *args, **kwargs):
         """
         try:
             return self.sink.add(*args, **kwargs)
-        except AttributeError as ex:
+        except AttributeError:
             msg = "%s has no data sink to put objects in"
             raise AttributeError(msg % self.__class__.__name__)
 
diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index 118db629..bc675b3e 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -1,15 +1,13 @@
 from typing import Any
 
-from sqlalchemy import (
-    create_engine, Boolean, Float, Integer, LargeBinary, Text, TIMESTAMP,
-)
+from sqlalchemy import Boolean, Float, Integer, Text, create_engine
 from sqlalchemy_utils import create_database, database_exists, drop_database
 
 from stix2.base import (
-    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
-    _STIXBase,
+    _DomainObject, _MetaObject, _Observable, _RelationshipObject,
 )
 
+
 class DatabaseBackend:
     def __init__(self, database_connection_url, force_recreate=False, **kwargs: Any):
         self.database_connection_url = database_connection_url
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index e80fd1bd..0d24056f 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -1,23 +1,13 @@
 import os
 from typing import Any
 
-from sqlalchemy import (
-    TIMESTAMP, LargeBinary, Text,
-)
+from sqlalchemy import TIMESTAMP, LargeBinary, Text
 from sqlalchemy.schema import CreateSchema
 
 from stix2.base import (
-    _DomainObject, _Extension, _MetaObject, _Observable, _RelationshipObject,
-    _STIXBase,
+    _DomainObject, _MetaObject, _Observable, _RelationshipObject,
 )
 from stix2.datastore.relational_db.utils import schema_for
-from stix2.properties import (
-    BinaryProperty, BooleanProperty, DictionaryProperty,
-    EmbeddedObjectProperty, EnumProperty, ExtensionsProperty, FloatProperty,
-    HashesProperty, HexProperty, IDProperty, IntegerProperty, ListProperty,
-    ObjectReferenceProperty, Property, ReferenceProperty, StringProperty,
-    TimestampProperty, TypeProperty,
-)
 
 from .database_backend_base import DatabaseBackend
 
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index ff308517..a850755b 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -248,7 +248,11 @@ def generate_insert_information(   # noqa: F811
         return insert_statements
     else:
         if db_backend.array_allowed():
-            return {name: stix_object[name]}
+            if isinstance(self.contained, HexProperty):
+                return {name: [bytes.fromhex(x) for x in stix_object[name]]}
+            else:
+                return {name: stix_object[name]}
+
         else:
             insert_statements = list()
             table = data_sink.tables_dictionary[
@@ -258,13 +262,14 @@ def generate_insert_information(   # noqa: F811
                 )
             ]
             for elem in stix_object[name]:
-                bindings = {"id": stix_object["id"], name: elem}
+                bindings = {
+                    "id": stix_object["id"],
+                    name: bytes.fromhex(elem) if isinstance(self.contained, HexProperty) else elem,
+                }
                 insert_statements.append(insert(table).values(bindings))
             return insert_statements
 
 
-
-
 @add_method(ReferenceProperty)
 def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
     return {name: stix_object[name]}
@@ -300,7 +305,6 @@ def generate_insert_for_array_in_table(table, values, foreign_key_value, column_
 
 
 def generate_insert_for_external_references(data_sink, stix_object):
-    db_backend = data_sink.db_backend
     insert_statements = list()
     next_id = None
     object_table = data_sink.tables_dictionary["common.external_references"]
@@ -395,7 +399,7 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, stix_type_
     core_insert_statement = insert(core_table).values(core_bindings)
     insert_statements.append(core_insert_statement)
 
-    if "labels" in stix_object:
+    if "labels" in stix_object and "labels" in child_table_properties:
         label_table_name = canonicalize_table_name(core_table.name + "_labels", data_sink.db_backend.schema_for_core())
         labels_table = data_sink.tables_dictionary[label_table_name]
         insert_statements.extend(
@@ -403,12 +407,15 @@ def generate_insert_for_core(data_sink, stix_object, core_properties, stix_type_
                 labels_table,
                 stix_object["labels"],
                 stix_object["id"],
-                column_name="label"
-            ))
+                column_name="label",
+            ),
+        )
 
     if "object_marking_refs" in stix_object:
-        object_marking_table_name = canonicalize_table_name("object_marking_refs",
-                                                            data_sink.db_backend.schema_for_core())
+        object_marking_table_name = canonicalize_table_name(
+            "object_marking_refs",
+            data_sink.db_backend.schema_for_core(),
+        )
         if stix_type_name != "sco":
             object_markings_ref_table = data_sink.tables_dictionary[object_marking_table_name + "_sdo"]
         else:
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 56cd32d0..2dc6009e 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -1,19 +1,14 @@
-from sqlalchemy import MetaData, create_engine, delete, select
-from sqlalchemy.schema import CreateSchema, CreateTable, Sequence
-from sqlalchemy_utils import create_database, database_exists, drop_database
+from sqlalchemy import MetaData, delete
+from sqlalchemy.schema import CreateTable, Sequence
 
-from stix2.base import (
-    _DomainObject, _MetaObject, _Observable, _RelationshipObject, _STIXBase,
-)
+from stix2.base import _STIXBase
 from stix2.datastore import DataSink, DataSource, DataStoreMixin
 from stix2.datastore.relational_db.input_creation import (
     generate_insert_for_object,
 )
 from stix2.datastore.relational_db.query import read_object
 from stix2.datastore.relational_db.table_creation import create_table_objects
-from stix2.datastore.relational_db.utils import (
-    canonicalize_table_name, schema_for, table_name_for,
-)
+from stix2.datastore.relational_db.utils import canonicalize_table_name
 from stix2.parsing import parse
 
 
@@ -88,6 +83,7 @@ def __init__(
             source=RelationalDBSource(
                 db_backend,
                 metadata=self.metadata,
+                allow_custom=allow_custom,
             ),
             sink=RelationalDBSink(
                 db_backend,
@@ -190,9 +186,10 @@ def next_id(self):
         with self.db_backend.database_connection.begin() as trans:
             return trans.execute(self.sequence)
 
+
 class RelationalDBSource(DataSource):
     def __init__(
-        self, db_backend, *stix_object_classes, metadata=None,
+        self, db_backend, allow_custom, *stix_object_classes, metadata=None,
     ):
         """
         Initialize this source.  Only one of stix_object_classes and metadata
@@ -217,6 +214,8 @@ def __init__(
 
         self.db_backend = db_backend
 
+        self.allow_custom = allow_custom
+
         if metadata:
             self.metadata = metadata
         else:
diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 926803a2..8f200b36 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -64,7 +64,7 @@ def malware_with_all_required_properties():
     ref2 = stix2.v21.ExternalReference(
         source_name="ACME Threat Intel",
         description="Threat report",
-        url="http://www.example.com/threat-report.pdf"
+        url="http://www.example.com/threat-report.pdf",
     )
     now = dt.datetime(2016, 5, 12, 8, 17, 27, tzinfo=pytz.utc)
 
@@ -203,10 +203,10 @@ def custom_obj():
 
 @stix2.CustomObject(
     "test-object", [
-        ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty()))
+        ("prop_name", stix2.properties.ListProperty(stix2.properties.BinaryProperty())),
     ],
     "extension-definition--15de9cdb-3515-4271-8479-8141154c5647",
-    is_sdo=True
+    is_sdo=True,
 )
 class TestClass:
     pass
@@ -215,53 +215,58 @@ class TestClass:
 def test_binary_list():
     return TestClass(prop_name=["AREi", "7t3M"])
 
+
 @stix2.CustomObject(
         "test2-object", [
-            ("prop_name", stix2.properties.ListProperty(
-                stix2.properties.HexProperty()
-            ))
+            (
+                "prop_name", stix2.properties.ListProperty(
+                    stix2.properties.HexProperty(),
+                ),
+            ),
         ],
         "extension-definition--15de9cdb-4567-4271-8479-8141154c5647",
-        is_sdo=True
-    )
-
+        is_sdo=True,
+)
 class Test2Class:
-        pass
+    pass
+
 
 def test_hex_list():
     return Test2Class(
-        prop_name=["1122", "fedc"]
+        prop_name=["1122", "fedc"],
     )
 
+
 @stix2.CustomObject(
         "test3-object", [
-            ("prop_name",
-                 stix2.properties.DictionaryProperty(
-                     valid_types=[
-                         stix2.properties.IntegerProperty,
-                         stix2.properties.FloatProperty,
-                         stix2.properties.StringProperty
-                     ]
-                 )
-             )
+            (
+                "prop_name",
+                stix2.properties.DictionaryProperty(
+                    valid_types=[
+                        stix2.properties.IntegerProperty,
+                        stix2.properties.FloatProperty,
+                        stix2.properties.StringProperty,
+                    ],
+                ),
+            ),
         ],
         "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
-        is_sdo=True
-    )
+        is_sdo=True,
+)
 class Test3Class:
     pass
 
 
 def test_dictionary():
     return Test3Class(
-        prop_name={"a": 1, "b": 2.3, "c": "foo"}
+        prop_name={"a": 1, "b": 2.3, "c": "foo"},
     )
 
 
 def main():
     store = RelationalDBStore(
         PostgresBackend("postgresql://localhost/stix-data-sink", force_recreate=True),
-        False,
+        True,
         None,
         True,
         print_sql=True,
@@ -269,17 +274,17 @@ def main():
 
     if store.sink.db_backend.database_exists:
 
-        # td = test_dictionary()
-        #
-        # store.add(td)
-        #
-        # th = test_hex_list()
-        #
-        # store.add(th)
-        #
-        # tb = test_binary_list()
-        #
-        # store.add(tb)
+        td = test_dictionary()
+
+        store.add(td)
+
+        th = test_hex_list()
+
+        store.add(th)
+
+        tb = test_binary_list()
+
+        store.add(tb)
 
         co = custom_obj()
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index d1feeddb..d101db48 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -1,8 +1,8 @@
 # from collections import OrderedDict
 
 from sqlalchemy import (  # create_engine,; insert,
-    ARRAY, TIMESTAMP, Boolean, CheckConstraint, Column, ForeignKey,
-    Integer, Table, Text, UniqueConstraint
+    ARRAY, CheckConstraint, Column, ForeignKey, Integer, Table, Text,
+    UniqueConstraint,
 )
 
 from stix2.datastore.relational_db.add_method import add_method
@@ -27,7 +27,7 @@ def create_array_column(property_name, contained_sql_type, optional):
         property_name,
         ARRAY(contained_sql_type),
         CheckConstraint(f"{property_name} IS NULL or array_length({property_name}, 1) IS NOT NULL"),
-        nullable=optional
+        nullable=optional,
     )
 
 
@@ -183,11 +183,12 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
         columns.append(create_array_column("selectors", db_backend.determine_sql_type_for_string_property(), False))
 
     else:
-        columns.append(Column(
-                "selectors",
-                db_backend.determine_sql_type_for_key_as_int(),
-                unique=True
-            )
+        columns.append(
+            Column(
+                    "selectors",
+                    db_backend.determine_sql_type_for_key_as_int(),
+                    unique=True,
+            ),
         )
 
         child_columns = [
@@ -206,8 +207,12 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                 nullable=False,
             ),
         ]
-        tables.append(Table(canonicalize_table_name("granular_marking_" + sco_or_sdo + "_" + "selector"),
-                            metadata, *child_columns, schema=schema_name))
+        tables.append(
+            Table(
+                canonicalize_table_name("granular_marking_" + sco_or_sdo + "_" + "selector"),
+                metadata, *child_columns, schema=schema_name,
+            ),
+        )
     tables.append(
         Table(
             "granular_marking_" + sco_or_sdo,
@@ -218,8 +223,9 @@ def create_granular_markings_table(metadata, db_backend, sco_or_sdo):
                       OR
                       (lang IS NOT NULL AND marking_ref IS NULL)""",
             ),
-            schema=schema_name
-        ))
+            schema=schema_name,
+        ),
+    )
     return tables
 
 
@@ -279,12 +285,16 @@ def create_core_table(metadata, db_backend, stix_type_name):
         if db_backend.array_allowed():
             columns.append(create_array_column("labels", db_backend.determine_sql_type_for_string_property(), True))
         else:
-            tables.append(create_array_child_table(metadata,
-                                                   db_backend,
-                                                   table_name,
-                                                   "_labels",
-                                                   "label",
-                                                   db_backend.determine_sql_type_for_string_property()))
+            tables.append(
+                create_array_child_table(
+                    metadata,
+                    db_backend,
+                    table_name,
+                    "_labels",
+                    "label",
+                    db_backend.determine_sql_type_for_string_property(),
+                ),
+            )
     else:
         columns.append(Column("defanged", db_backend.determine_sql_type_for_boolean_property(), default=False))
 
@@ -294,7 +304,8 @@ def create_core_table(metadata, db_backend, stix_type_name):
             metadata,
             *columns,
             schema=db_backend.schema_for_core(),
-        ))
+        ),
+    )
     return tables
 
 
@@ -422,9 +433,11 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
             else:
                 contained_class = self.valid_types[0].contained
                 columns.append(
-                    create_array_column("value",
-                                        contained_class.determine_sql_type(db_backend),
-                                        False)
+                    create_array_column(
+                        "value",
+                        contained_class.determine_sql_type(db_backend),
+                        False,
+                    ),
                 )
         else:
             for column_type in self.valid_types:
@@ -586,6 +599,7 @@ def generate_table_information(self, name, db_backend, **kwargs):  # noqa: F811
 @add_method(ListProperty)
 def generate_table_information(self, name, db_backend, metadata, schema_name, table_name, **kwargs):  # noqa: F811
     is_extension = kwargs.get('is_extension')
+    is_embedded_object = kwargs.get('is_embedded_object')
     tables = list()
     # handle more complex embedded object before deciding if the ARRAY type is usable
     if isinstance(self.contained, EmbeddedObjectProperty):
@@ -637,13 +651,23 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
                 schema_name,
             ),
         ]
-    elif ((isinstance(self.contained, (StringProperty, IntegerProperty, FloatProperty)) and not db_backend.array_allowed()) or
+    elif ((
+        isinstance(
+            self.contained,
+            (BinaryProperty, BooleanProperty, StringProperty, IntegerProperty, FloatProperty, HexProperty),
+        ) and
+        not db_backend.array_allowed()
+    ) or
           isinstance(self.contained, EnumProperty)):
         columns = list()
+        if is_embedded_object:
+            id_type = db_backend.determine_sql_type_for_key_as_int()
+        else:
+            id_type = db_backend.determine_sql_type_for_key_as_id()
         columns.append(
             Column(
                 "id",
-                self.contained.determine_sql_type(db_backend),
+                id_type,
                 ForeignKey(
                     canonicalize_table_name(table_name, schema_name) + ".id",
                     ondelete="CASCADE",
diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index 92806b38..ca387e46 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -1,9 +1,6 @@
 from collections.abc import Iterable, Mapping
 
 import inflection
-from sqlalchemy import (  # create_engine,; insert,
-    TIMESTAMP, Boolean, Float, Integer, LargeBinary, Text,
-)
 
 from stix2.properties import (
     BinaryProperty, BooleanProperty, FloatProperty, HexProperty,

From 1af893ea472cf988ff19703117b6f9ca8015600b Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Tue, 19 Nov 2024 00:05:43 -0500
Subject: [PATCH 129/132] Add a missing "return" to
 determine_sql_type_from_stix()

---
 stix2/datastore/relational_db/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/utils.py b/stix2/datastore/relational_db/utils.py
index ca387e46..65257426 100644
--- a/stix2/datastore/relational_db/utils.py
+++ b/stix2/datastore/relational_db/utils.py
@@ -147,7 +147,7 @@ def determine_sql_type_from_stix(cls_or_inst, db_backend):  # noqa: F811
     elif is_class_or_instance(cls_or_inst, StringProperty):
         return db_backend.determine_sql_type_for_string_property()
     elif is_class_or_instance(cls_or_inst, ReferenceProperty):
-        db_backend.determine_sql_type_for_reference_property()
+        return db_backend.determine_sql_type_for_reference_property()
     elif is_class_or_instance(cls_or_inst, TimestampProperty):
         return db_backend.determine_sql_type_for_timestamp_property()
     elif is_class_or_instance(cls_or_inst, Property):

From 9b9ca635c3dbd7cedb4a0783d1ea4bbbd084a15c Mon Sep 17 00:00:00 2001
From: Michael Chisholm <chisholm@mitre.org>
Date: Tue, 19 Nov 2024 17:30:34 -0500
Subject: [PATCH 130/132] Add support for db backends and dbs which don't
 support array column types, to the relational data source

---
 stix2/datastore/relational_db/query.py        | 150 ++++++++++++++----
 .../datastore/relational_db/relational_db.py  |   6 +-
 .../test/v21/test_datastore_relational_db.py  |  10 +-
 3 files changed, 129 insertions(+), 37 deletions(-)

diff --git a/stix2/datastore/relational_db/query.py b/stix2/datastore/relational_db/query.py
index f223852b..637b66e9 100644
--- a/stix2/datastore/relational_db/query.py
+++ b/stix2/datastore/relational_db/query.py
@@ -99,6 +99,22 @@ def _read_simple_properties(stix_id, core_table, type_table, conn):
     return obj_dict
 
 
+def _read_simple_array(fk_id, elt_column_name, array_table, conn):
+    """
+    Read array elements from a given table.
+
+    :param fk_id: A foreign key value used to find the correct array elements
+    :param elt_column_name: The name of the table column which contains the
+        array elements
+    :param array_table: A SQLAlchemy Table object containing the array data
+    :param conn: An SQLAlchemy DB connection
+    :return: The array, as a list
+    """
+    stmt = sa.select(array_table.c[elt_column_name]).where(array_table.c.id == fk_id)
+    refs = conn.scalars(stmt).all()
+    return refs
+
+
 def _read_hashes(fk_id, hashes_table, conn):
     """
     Read hashes from a table.
@@ -178,7 +194,7 @@ def _read_object_marking_refs(stix_id, stix_type_class, metadata, conn):
     return refs
 
 
-def _read_granular_markings(stix_id, stix_type_class, metadata, conn):
+def _read_granular_markings(stix_id, stix_type_class, metadata, conn, db_backend):
     """
     Read granular markings from one of a couple special tables in the common
     schema.
@@ -189,6 +205,8 @@ def _read_granular_markings(stix_id, stix_type_class, metadata, conn):
     :param metadata: SQLAlchemy Metadata object containing all the table
         information
     :param conn: An SQLAlchemy DB connection
+    :param db_backend: A backend object with information about how data is
+        stored in the database
     :return: Granular markings as a list of dicts
     """
 
@@ -200,30 +218,43 @@ def _read_granular_markings(stix_id, stix_type_class, metadata, conn):
 
     marking_table = metadata.tables["common." + marking_table_name]
 
-    stmt = sa.select(
-        marking_table.c.lang,
-        marking_table.c.marking_ref,
-        marking_table.c.selectors,
-    ).where(marking_table.c.id == stix_id)
-
-    marking_dicts = conn.execute(stmt).mappings().all()
-    return marking_dicts
+    if db_backend.array_allowed():
+        # arrays allowed: everything combined in the same table
+        stmt = sa.select(
+            marking_table.c.lang,
+            marking_table.c.marking_ref,
+            marking_table.c.selectors,
+        ).where(marking_table.c.id == stix_id)
 
+        marking_dicts = conn.execute(stmt).mappings().all()
 
-def _read_simple_array(fk_id, elt_column_name, array_table, conn):
-    """
-    Read array elements from a given table.
+    else:
+        # arrays not allowed: selectors are in their own table
+        stmt = sa.select(
+            marking_table.c.lang,
+            marking_table.c.marking_ref,
+            marking_table.c.selectors,
+        ).where(marking_table.c.id == stix_id)
+
+        marking_dicts = list(conn.execute(stmt).mappings())
+
+        for idx, marking_dict in enumerate(marking_dicts):
+            # make a mutable shallow-copy of the row mapping
+            marking_dicts[idx] = marking_dict = dict(marking_dict)
+            selector_id = marking_dict.pop("selectors")
+
+            selector_table_name = f"{marking_table.fullname}_selector"
+            selector_table = metadata.tables[selector_table_name]
+
+            selectors = _read_simple_array(
+                selector_id,
+                "selector",
+                selector_table,
+                conn
+            )
+            marking_dict["selectors"] = selectors
 
-    :param fk_id: A foreign key value used to find the correct array elements
-    :param elt_column_name: The name of the table column which contains the
-        array elements
-    :param array_table: A SQLAlchemy Table object containing the array data
-    :param conn: An SQLAlchemy DB connection
-    :return: The array, as a list
-    """
-    stmt = sa.select(array_table.c[elt_column_name]).where(array_table.c.id == fk_id)
-    refs = conn.scalars(stmt).all()
-    return refs
+    return marking_dicts
 
 
 def _read_kill_chain_phases(stix_id, type_table, metadata, conn):
@@ -437,10 +468,26 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
             ref_table = metadata.tables[ref_table_name]
             prop_value = _read_simple_array(obj_id, "ref_id", ref_table, conn)
 
-        elif isinstance(prop_instance.contained, stix2.properties.EnumProperty):
-            enum_table_name = f"{obj_table.fullname}_{prop_name}"
-            enum_table = metadata.tables[enum_table_name]
-            prop_value = _read_simple_array(obj_id, prop_name, enum_table, conn)
+        elif isinstance(prop_instance.contained, (
+            # Most of these list-of-simple-type cases would occur when array
+            # columns are disabled.
+            stix2.properties.BinaryProperty,
+            stix2.properties.BooleanProperty,
+            stix2.properties.EnumProperty,
+            stix2.properties.HexProperty,
+            stix2.properties.IntegerProperty,
+            stix2.properties.FloatProperty,
+            stix2.properties.StringProperty,
+            stix2.properties.TimestampProperty,
+        )):
+            array_table_name = f"{obj_table.fullname}_{prop_name}"
+            array_table = metadata.tables[array_table_name]
+            prop_value = _read_simple_array(
+                obj_id,
+                prop_name,
+                array_table,
+                conn
+            )
 
         elif isinstance(prop_instance.contained, stix2.properties.EmbeddedObjectProperty):
             join_table_name = f"{obj_table.fullname}_{prop_name}"
@@ -494,7 +541,16 @@ def _read_complex_property_value(obj_id, prop_name, prop_instance, obj_table, me
     return prop_value
 
 
-def _read_complex_top_level_property_value(stix_id, stix_type_class, prop_name, prop_instance, type_table, metadata, conn):
+def _read_complex_top_level_property_value(
+    stix_id,
+    stix_type_class,
+    prop_name,
+    prop_instance,
+    type_table,
+    metadata,
+    conn,
+    db_backend
+):
     """
     Read property values which require auxiliary tables to store.  These
     require a lot of special cases.  This function has additional support for
@@ -511,6 +567,8 @@ def _read_complex_top_level_property_value(stix_id, stix_type_class, prop_name,
     :param metadata: SQLAlchemy Metadata object containing all the table
         information
     :param conn: An SQLAlchemy DB connection
+    :param db_backend: A backend object with information about how data is
+        stored in the database
     :return: The property value
     """
 
@@ -519,19 +577,44 @@ def _read_complex_top_level_property_value(stix_id, stix_type_class, prop_name,
         prop_value = _read_external_references(stix_id, metadata, conn)
 
     elif prop_name == "object_marking_refs":
-        prop_value = _read_object_marking_refs(stix_id, stix_type_class, metadata, conn)
+        prop_value = _read_object_marking_refs(
+            stix_id,
+            stix_type_class,
+            metadata,
+            conn
+        )
 
     elif prop_name == "granular_markings":
-        prop_value = _read_granular_markings(stix_id, stix_type_class, metadata, conn)
+        prop_value = _read_granular_markings(
+            stix_id,
+            stix_type_class,
+            metadata,
+            conn,
+            db_backend
+        )
+
+    # Will apply when array columns are unsupported/disallowed by the backend
+    elif prop_name == "labels":
+        label_table = metadata.tables[
+            f"common.core_{stix_type_class.name.lower()}_labels"
+        ]
+        prop_value = _read_simple_array(stix_id, "label", label_table, conn)
 
     else:
         # Other properties use specific table patterns depending on property type
-        prop_value = _read_complex_property_value(stix_id, prop_name, prop_instance, type_table, metadata, conn)
+        prop_value = _read_complex_property_value(
+            stix_id,
+            prop_name,
+            prop_instance,
+            type_table,
+            metadata,
+            conn
+        )
 
     return prop_value
 
 
-def read_object(stix_id, metadata, conn):
+def read_object(stix_id, metadata, conn, db_backend):
     """
     Read a STIX object from the database, identified by a STIX ID.
 
@@ -539,6 +622,8 @@ def read_object(stix_id, metadata, conn):
     :param metadata: SQLAlchemy Metadata object containing all the table
         information
     :param conn: An SQLAlchemy DB connection
+    :param db_backend: A backend object with information about how data is
+        stored in the database
     :return: A STIX object
     """
     _check_support(stix_id)
@@ -554,7 +639,7 @@ def read_object(stix_id, metadata, conn):
     if type_table.schema == "common":
         # Applies to extension-definition SMO, whose data is stored in the
         # common schema; it does not get its own.  This type class is used to
-        # determine which markings tables to use; its markings are
+        # determine which common tables to use; its markings are
         # in the *_sdo tables.
         stix_type_class = stix2.utils.STIXTypeClass.SDO
     else:
@@ -578,6 +663,7 @@ def read_object(stix_id, metadata, conn):
                 type_table,
                 metadata,
                 conn,
+                db_backend
             )
 
             if prop_value is not None:
diff --git a/stix2/datastore/relational_db/relational_db.py b/stix2/datastore/relational_db/relational_db.py
index 2dc6009e..ee0a312f 100644
--- a/stix2/datastore/relational_db/relational_db.py
+++ b/stix2/datastore/relational_db/relational_db.py
@@ -195,10 +195,11 @@ def __init__(
         Initialize this source.  Only one of stix_object_classes and metadata
         should be given: if the latter is given, assume table schemas are
         already created.  Instances of this class do not create the actual
-        database tables; see the source/sink for that.
+        database tables; see the store/sink for that.
 
         Args:
-            database_connection_or_url: An SQLAlchemy engine object, or URL
+            db_backend: A database backend object
+            allow_custom: TODO: unused so far
             *stix_object_classes: STIX object classes to map into table schemas.
                 This can be used to limit which schemas are created, if one is
                 only working with a subset of STIX types.  If not given,
@@ -230,6 +231,7 @@ def get(self, stix_id, version=None, _composite_filters=None):
                 stix_id,
                 self.metadata,
                 conn,
+                self.db_backend,
             )
 
         return stix_obj
diff --git a/stix2/test/v21/test_datastore_relational_db.py b/stix2/test/v21/test_datastore_relational_db.py
index 5887c066..316303db 100644
--- a/stix2/test/v21/test_datastore_relational_db.py
+++ b/stix2/test/v21/test_datastore_relational_db.py
@@ -7,6 +7,9 @@
 
 import stix2
 from stix2.datastore import DataSourceError
+from stix2.datastore.relational_db.database_backends.postgres_backend import (
+    PostgresBackend,
+)
 from stix2.datastore.relational_db.relational_db import RelationalDBStore
 import stix2.properties
 import stix2.registry
@@ -15,7 +18,7 @@
 _DB_CONNECT_URL = f"postgresql://{os.getenv('POSTGRES_USER', 'postgres')}:{os.getenv('POSTGRES_PASSWORD', 'postgres')}@0.0.0.0:5432/postgres"
 
 store = RelationalDBStore(
-    _DB_CONNECT_URL,
+    PostgresBackend(_DB_CONNECT_URL, True),
     True,
     None,
     False,
@@ -878,7 +881,7 @@ def test_property(object_variation):
     ensure schemas can be created and values can be stored and retrieved.
     """
     rdb_store = RelationalDBStore(
-        _DB_CONNECT_URL,
+        PostgresBackend(_DB_CONNECT_URL, True),
         True,
         None,
         True,
@@ -918,7 +921,7 @@ def test_dictionary_property_complex():
         )
 
         rdb_store = RelationalDBStore(
-            _DB_CONNECT_URL,
+            PostgresBackend(_DB_CONNECT_URL, True),
             True,
             None,
             True,
@@ -934,6 +937,7 @@ def test_dictionary_property_complex():
 def test_extension_definition():
     obj = stix2.ExtensionDefinition(
         created_by_ref="identity--8a5fb7e4-aabe-4635-8972-cbcde1fa4792",
+        labels=["label1", "label2"],
         name="test",
         schema="a schema",
         version="1.2.3",

From 6465249056686f2d4c5fc26a770616ff0d1c3a80 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Wed, 20 Nov 2024 09:52:09 -0500
Subject: [PATCH 131/132] fix for list of timestamps

---
 stix2/datastore/relational_db/relational_db_testing.py | 5 +++++
 stix2/datastore/relational_db/table_creation.py        | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/stix2/datastore/relational_db/relational_db_testing.py b/stix2/datastore/relational_db/relational_db_testing.py
index 8f200b36..a6376fd2 100644
--- a/stix2/datastore/relational_db/relational_db_testing.py
+++ b/stix2/datastore/relational_db/relational_db_testing.py
@@ -249,6 +249,10 @@ def test_hex_list():
                     ],
                 ),
             ),
+            (
+                "list_of_timestamps",
+                stix2.properties.ListProperty(stix2.properties.TimestampProperty()),
+            ),
         ],
         "extension-definition--15de9cdb-1234-4271-8479-8141154c5647",
         is_sdo=True,
@@ -260,6 +264,7 @@ class Test3Class:
 def test_dictionary():
     return Test3Class(
         prop_name={"a": 1, "b": 2.3, "c": "foo"},
+        list_of_timestamps={ "2016-05-12T08:17:27.000Z", "2024-05-12T08:17:27.000Z"}
     )
 
 
diff --git a/stix2/datastore/relational_db/table_creation.py b/stix2/datastore/relational_db/table_creation.py
index d101db48..32965f02 100644
--- a/stix2/datastore/relational_db/table_creation.py
+++ b/stix2/datastore/relational_db/table_creation.py
@@ -654,7 +654,7 @@ def generate_table_information(self, name, db_backend, metadata, schema_name, ta
     elif ((
         isinstance(
             self.contained,
-            (BinaryProperty, BooleanProperty, StringProperty, IntegerProperty, FloatProperty, HexProperty),
+            (BinaryProperty, BooleanProperty, StringProperty, IntegerProperty, FloatProperty, HexProperty, TimestampProperty),
         ) and
         not db_backend.array_allowed()
     ) or

From 0e3bf331dd0bff9fdde2e2d6b2b417392ffbaf91 Mon Sep 17 00:00:00 2001
From: Rich Piazza <rpiazza@mitre.org>
Date: Thu, 21 Nov 2024 17:01:19 -0500
Subject: [PATCH 132/132] handle-hex-better

---
 .../database_backends/database_backend_base.py             | 7 +++++++
 .../relational_db/database_backends/postgres_backend.py    | 5 +++--
 stix2/datastore/relational_db/input_creation.py            | 7 +++----
 3 files changed, 13 insertions(+), 6 deletions(-)

diff --git a/stix2/datastore/relational_db/database_backends/database_backend_base.py b/stix2/datastore/relational_db/database_backends/database_backend_base.py
index bc675b3e..bb03f55f 100644
--- a/stix2/datastore/relational_db/database_backends/database_backend_base.py
+++ b/stix2/datastore/relational_db/database_backends/database_backend_base.py
@@ -105,3 +105,10 @@ def determine_sql_type_for_key_as_id():  # noqa: F811
     @staticmethod
     def array_allowed():
         return False
+
+    def generate_value(self, stix_type, value):
+        sql_type = stix_type.determine_sql_type(self)
+        if sql_type == self.determine_sql_type_for_string_property():
+            return value
+        elif sql_type == self.determine_sql_type_for_hex_property():
+            return bytes.fromhex(value)
diff --git a/stix2/datastore/relational_db/database_backends/postgres_backend.py b/stix2/datastore/relational_db/database_backends/postgres_backend.py
index 0d24056f..ca501dfb 100644
--- a/stix2/datastore/relational_db/database_backends/postgres_backend.py
+++ b/stix2/datastore/relational_db/database_backends/postgres_backend.py
@@ -50,11 +50,12 @@ def schema_for_core():
 
     @staticmethod
     def determine_sql_type_for_binary_property():  # noqa: F811
-        return Text
+        return PostgresBackend.determine_sql_type_for_string_property()
 
     @staticmethod
     def determine_sql_type_for_hex_property():  # noqa: F811
-        return LargeBinary
+        # return LargeBinary
+        return PostgresBackend.determine_sql_type_for_string_property()
 
     @staticmethod
     def determine_sql_type_for_timestamp_property():  # noqa: F811
diff --git a/stix2/datastore/relational_db/input_creation.py b/stix2/datastore/relational_db/input_creation.py
index a850755b..6bc04aa2 100644
--- a/stix2/datastore/relational_db/input_creation.py
+++ b/stix2/datastore/relational_db/input_creation.py
@@ -140,9 +140,8 @@ def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F81
 
 
 @add_method(HexProperty)
-def generate_insert_information(self, name, stix_object, **kwargs):  # noqa: F811
-    v = bytes.fromhex(stix_object[name])
-    return {name: v}
+def generate_insert_information(self, name, stix_object, data_sink, **kwargs):  # noqa: F811
+    return {name: data_sink.db_backend.generate_value(self, stix_object[name])}
 
 
 def generate_insert_for_hashes(
@@ -249,7 +248,7 @@ def generate_insert_information(   # noqa: F811
     else:
         if db_backend.array_allowed():
             if isinstance(self.contained, HexProperty):
-                return {name: [bytes.fromhex(x) for x in stix_object[name]]}
+                return {name: [data_sink.db_backend.generate_value(self.contained, x) for x in stix_object[name]]}
             else:
                 return {name: stix_object[name]}