From d7a51bfdc6094cedbf2f255a0f48f8531ec016ed Mon Sep 17 00:00:00 2001 From: Dat Nguyen Date: Sun, 28 Jul 2024 13:30:39 +0700 Subject: [PATCH] test: add unittest --- dbterd/adapters/algos/semantic.py | 52 ++-- tests/unit/adapters/algos/__init__.py | 288 ++++++++++++++++++ tests/unit/adapters/algos/test_semantic.py | 78 +++++ .../adapters/algos/test_test_relationship.py | 239 +-------------- 4 files changed, 410 insertions(+), 247 deletions(-) create mode 100644 tests/unit/adapters/algos/test_semantic.py diff --git a/dbterd/adapters/algos/semantic.py b/dbterd/adapters/algos/semantic.py index 1263a44..7c0d91a 100644 --- a/dbterd/adapters/algos/semantic.py +++ b/dbterd/adapters/algos/semantic.py @@ -8,14 +8,14 @@ def parse_metadata(data, **kwargs) -> Tuple[List[Table], List[Ref]]: - raise NotImplementedError() + raise NotImplementedError() # pragma: no cover def parse( manifest: Manifest, catalog: Union[str, Catalog], **kwargs ) -> Tuple[List[Table], List[Ref]]: # Parse metadata - if catalog == "metadata": + if catalog == "metadata": # pragma: no cover return parse_metadata(data=manifest, **kwargs) # Parse Table @@ -56,7 +56,7 @@ def find_related_nodes_by_id( List[str]: Manifest nodes' unique ID """ found_nodes = [node_unique_id] - if type == "metadata": + if type == "metadata": # pragma: no cover return found_nodes # not supported yet, returned input only entities = _get_linked_semantic_entities(manifest=manifest) @@ -79,18 +79,20 @@ def _get_relationships(manifest: Manifest, **kwargs) -> List[Ref]: List[Ref]: List of parsed relationship """ entities = _get_linked_semantic_entities(manifest=manifest) - return [ - Ref( - name=primary_entity.semantic_model, - table_map=(primary_entity.model, foreign_entity.model), - column_map=( - foreign_entity.column_name, - primary_entity.column_name, - ), - type=primary_entity.relationship_type, - ) - for foreign_entity, primary_entity in entities - ] + return base.get_unique_refs( + refs=[ + Ref( + name=primary_entity.semantic_model, + table_map=(primary_entity.model, foreign_entity.model), + column_map=( + primary_entity.column_name, + foreign_entity.column_name, + ), + type=primary_entity.relationship_type, + ) + for foreign_entity, primary_entity in entities + ] + ) def _get_linked_semantic_entities( @@ -129,20 +131,34 @@ def _get_semantic_entities( semantic_entities = [] for x in _get_semantic_nodes(manifest=manifest): - for e in manifest.semantic_models[x].entities: + semantic_node = manifest.semantic_models[x] + for e in semantic_node.entities: if e.type.value in [PK, FK]: semantic_entities.append( SemanticEntity( semantic_model=x, - model=manifest.semantic_models[x].depends_on.nodes[0], + model=semantic_node.depends_on.nodes[0], entity_name=e.name, entity_type=e.type.value, column_name=e.expr or e.name, - relationship_type=manifest.semantic_models[x].config.meta.get( + relationship_type=semantic_node.config.meta.get( TEST_META_RELATIONSHIP_TYPE, "" ), ) ) + if semantic_node.primary_entity: + semantic_entities.append( + SemanticEntity( + semantic_model=x, + model=semantic_node.depends_on.nodes[0], + entity_name=semantic_node.primary_entity, + entity_type=PK, + column_name=semantic_node.primary_entity, + relationship_type=semantic_node.config.meta.get( + TEST_META_RELATIONSHIP_TYPE, "" + ), + ) + ) return ( [x for x in semantic_entities if x.entity_type == FK], diff --git a/tests/unit/adapters/algos/__init__.py b/tests/unit/adapters/algos/__init__.py index e69de29..5369cb1 100644 --- a/tests/unit/adapters/algos/__init__.py +++ b/tests/unit/adapters/algos/__init__.py @@ -0,0 +1,288 @@ +from dataclasses import dataclass, field + + +@dataclass +class DummyManifestV6: + compiled_sql: str = "compiled_sql" + + +@dataclass +class DummyManifestV7: + compiled_code: str = "compiled_code" + + +@dataclass +class DummyManifestError: + raw_sql: str = "raw_sql" + + +@dataclass +class DummyManifestHasColumns: + columns = dict({"col1": None, "col2": None}) + database = "database_dummy" + schema = "schema_dummy" + + +@dataclass +class ManifestNodeTestMetaData: + kwargs: dict + + +@dataclass +class ManifestNodeDependsOn: + nodes: list = field(default_factory=list) + + +@dataclass +class ManifestNode: + test_metadata: ManifestNodeTestMetaData + meta: dict + columns: dict + raw_sql: str = "" + database: str = "" + schema_: str = "" + depends_on: ManifestNodeDependsOn = field(default_factory=ManifestNodeDependsOn) + description: str = "" + + +@dataclass +class SemanticModelEntityType: + value: str + + +@dataclass +class SemanticModelEntity: + name: str + type: SemanticModelEntityType + expr: str + + +@dataclass +class NodeConfig: + meta: dict = field(default_factory=dict) + + +@dataclass +class SemanticModel: + entities: list = field(default_factory=list) + depends_on: ManifestNodeDependsOn = field(default_factory=ManifestNodeDependsOn) + primary_entity: str = None + config: NodeConfig = field(default_factory=NodeConfig) + + +@dataclass +class ManifestExposureNode: + depends_on: ManifestNodeDependsOn + + +@dataclass +class ManifestNodeColumn: + name: str + data_type: str = "unknown" + description: str = "" + + +@dataclass +class DummyManifestRel: + semantic_models = { + "semantic_model.dbt_resto.sm1": SemanticModel( + entities=[ + SemanticModelEntity( + name="id1", type=SemanticModelEntityType(value="primary"), expr=None + ) + ], + depends_on=ManifestNodeDependsOn(nodes=["model.dbt_resto.table1"]), + primary_entity=None, + ), + "semantic_model.dbt_resto.sm2": SemanticModel( + entities=[ + SemanticModelEntity( + name="id1", + type=SemanticModelEntityType(value="foreign"), + expr="id2", + ) + ], + depends_on=ManifestNodeDependsOn(nodes=["model.dbt_resto.table2"]), + primary_entity="id2", + ), + "semantic_model.dbt_resto.smx": SemanticModel( + entities=[ + SemanticModelEntity( + name="pkx", type=SemanticModelEntityType(value="primary"), expr=None + ), + SemanticModelEntity( + name="id1", type=SemanticModelEntityType(value="foreign"), expr="x" + ), + ], + depends_on=ManifestNodeDependsOn(nodes=["model.dbt_resto.tablex"]), + primary_entity=None, + ), + } + nodes = { + "test.dbt_resto.relationships_table1": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('table2')"} + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] + ), + ), + "test.dbt_resto.relationships_table2": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('table2')"} + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] + ), + ), + "test.dbt_resto.relationships_table3": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('tabley')"} + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.tabley", "model.dbt_resto.tablex"] + ), + ), + "test.dbt_resto.relationships_tablex": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "x", "field": "y", "to": "ref('y')"} + ), + meta={"ignore_in_erd": 1}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.y", "model.dbt_resto.x"] + ), + ), + "test.dbt_resto.foreign_key_table1": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={ + "column_name": "f1", + "pk_column_name": "f2", + "pk_table_name": "ref('table2')", + } + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] + ), + ), + "test.dbt_resto.relationships_table4": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('table-m2')"} + ), + meta={"relationship_type": "one-to-one"}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table-m2", "model.dbt_resto.table-m1"] + ), + ), + "test.dbt_resto.relationships_table1_reverse": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('table-r2')"} + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table-r1", "model.dbt_resto.table-r2"] + ), + ), + "test.dbt_resto.relationships_table1_recursive": ManifestNode( + test_metadata=ManifestNodeTestMetaData( + kwargs={"column_name": "f1", "field": "f2", "to": "ref('table1')"} + ), + meta={}, + columns={}, + depends_on=ManifestNodeDependsOn(nodes=["model.dbt_resto.table1"]), + ), + } + + +@dataclass +class DummyManifestTable: + nodes = { + "model.dbt_resto.table1": ManifestNode( + test_metadata=ManifestNodeTestMetaData(kwargs={}), + meta={}, + raw_sql="--raw_sql--", + database="--database--", + schema_="--schema--", + columns={}, + ), + "model.dbt_resto.table_dummy_columns": ManifestNode( + test_metadata=ManifestNodeTestMetaData(kwargs={}), + meta={}, + raw_sql="--raw_sql--", + database="--database--", + schema_="--schema--", + columns={}, + ), + "model.dbt_resto.table2": ManifestNode( + test_metadata=ManifestNodeTestMetaData(kwargs={}), + meta={}, + raw_sql="--raw_sql2--", + database="--database2--", + schema_="--schema2--", + columns={ + "name2": ManifestNodeColumn(name="name2"), + "name3": ManifestNodeColumn(name="name3"), + }, + ), + } + sources = { + "source.dummy.source_table": ManifestNode( + test_metadata=ManifestNodeTestMetaData(kwargs={}), + meta={}, + database="--database--", + schema_="--schema--", + columns={ + "name1": ManifestNodeColumn(name="name1"), + "name2": ManifestNodeColumn(name="name2"), + }, + ), + } + + +@dataclass +class DummyManifestWithExposure: + exposures = { + "exposure.dbt_resto.dummy": ManifestExposureNode( + depends_on=ManifestNodeDependsOn( + nodes=["model.dbt_resto.table1", "model.dbt_resto.table2"] + ), + ) + } + + +@dataclass +class CatalogNode: + columns: dict + + +@dataclass +class CatalogNodeColumn: + type: str + comment: str = "" + + +@dataclass +class DummyCatalogTable: + nodes = { + "model.dbt_resto.table1": CatalogNode( + columns={"name1": CatalogNodeColumn(type="--name1-type--")} + ), + "model.dbt_resto.table2": CatalogNode( + columns={"name3": CatalogNodeColumn(type="--name3-type--")} + ), + } + sources = { + "source.dummy.source_table": CatalogNode( + columns={"name1": CatalogNodeColumn(type="--name1-type--")} + ), + } diff --git a/tests/unit/adapters/algos/test_semantic.py b/tests/unit/adapters/algos/test_semantic.py new file mode 100644 index 0000000..bd66b05 --- /dev/null +++ b/tests/unit/adapters/algos/test_semantic.py @@ -0,0 +1,78 @@ +from unittest import mock +from unittest.mock import MagicMock + +import pytest + +from dbterd.adapters.algos import semantic +from dbterd.adapters.meta import Ref +from dbterd.adapters.targets import dbml as engine +from tests.unit.adapters.algos import DummyManifestRel, DummyManifestTable + + +class TestAlgoSemantic: + @pytest.mark.parametrize( + "manifest, expected", + [ + ( + DummyManifestRel(), + [ + Ref( + name="semantic_model.dbt_resto.sm1", + table_map=("model.dbt_resto.table1", "model.dbt_resto.table2"), + column_map=("id1", "id2"), + type="", + ), + Ref( + name="semantic_model.dbt_resto.sm1", + table_map=("model.dbt_resto.table1", "model.dbt_resto.tablex"), + column_map=("id1", "x"), + type="", + ), + ], + ), + (MagicMock(return_value={"semantic_models": {}, "nodes": {}}), []), + (DummyManifestTable(), []), + ], + ) + def test_get_relationships(self, manifest, expected): + assert semantic._get_relationships(manifest=manifest) == expected + + def test_find_related_nodes_by_id(self): + assert sorted(["model.dbt_resto.table1", "model.dbt_resto.table2"]) == sorted( + semantic.find_related_nodes_by_id( + manifest=DummyManifestRel(), node_unique_id="model.dbt_resto.table2" + ) + ) + assert sorted( + [ + "model.dbt_resto.table1", + "model.dbt_resto.table2", + "model.dbt_resto.tablex", + ] + ) == sorted( + semantic.find_related_nodes_by_id( + manifest=DummyManifestRel(), node_unique_id="model.dbt_resto.table1" + ) + ) + assert ["model.dbt_resto.not-exists"] == semantic.find_related_nodes_by_id( + manifest=DummyManifestRel(), node_unique_id="model.dbt_resto.not-exists" + ) + + def test_parse(self): + with mock.patch( + "dbterd.adapters.algos.base.get_tables", + ) as mock_get_tables: + with mock.patch( + "dbterd.adapters.algos.semantic._get_relationships", + ) as mock_get_relationships: + engine.parse( + manifest="--manifest--", + catalog="--catalog--", + select=[], + exclude=[], + resource_type=["model"], + algo="semantic", + omit_entity_name_quotes=False, + ) + mock_get_tables.assert_called_once() + mock_get_relationships.assert_called_once() diff --git a/tests/unit/adapters/algos/test_test_relationship.py b/tests/unit/adapters/algos/test_test_relationship.py index 13fca11..17d7b91 100644 --- a/tests/unit/adapters/algos/test_test_relationship.py +++ b/tests/unit/adapters/algos/test_test_relationship.py @@ -1,4 +1,3 @@ -from dataclasses import dataclass, field from unittest import mock from unittest.mock import MagicMock @@ -8,234 +7,16 @@ from dbterd.adapters.algos import base as base_algo from dbterd.adapters.algos import test_relationship from dbterd.adapters.meta import Column, Ref, Table - - -@dataclass -class DummyManifestV6: - compiled_sql: str = "compiled_sql" - - -@dataclass -class DummyManifestV7: - compiled_code: str = "compiled_code" - - -@dataclass -class DummyManifestError: - raw_sql: str = "raw_sql" - - -@dataclass -class DummyManifestHasColumns: - columns = dict({"col1": None, "col2": None}) - database = "database_dummy" - schema = "schema_dummy" - - -@dataclass -class ManifestNodeTestMetaData: - kwargs: dict - - -@dataclass -class ManifestNodeDependsOn: - nodes: list = field(default_factory=list) - - -@dataclass -class ManifestNode: - test_metadata: ManifestNodeTestMetaData - meta: dict - columns: dict - raw_sql: str = "" - database: str = "" - schema_: str = "" - depends_on: ManifestNodeDependsOn = field(default_factory=ManifestNodeDependsOn) - description: str = "" - - -@dataclass -class ManifestExposureNode: - depends_on: ManifestNodeDependsOn - - -@dataclass -class ManifestNodeColumn: - name: str - data_type: str = "unknown" - description: str = "" - - -@dataclass -class DummyManifestRel: - nodes = { - "test.dbt_resto.relationships_table1": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('table2')"} - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] - ), - ), - "test.dbt_resto.relationships_table2": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('table2')"} - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] - ), - ), - "test.dbt_resto.relationships_table3": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('tabley')"} - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.tabley", "model.dbt_resto.tablex"] - ), - ), - "test.dbt_resto.relationships_tablex": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "x", "field": "y", "to": "ref('y')"} - ), - meta={"ignore_in_erd": 1}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.y", "model.dbt_resto.x"] - ), - ), - "test.dbt_resto.foreign_key_table1": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={ - "column_name": "f1", - "pk_column_name": "f2", - "pk_table_name": "ref('table2')", - } - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table2", "model.dbt_resto.table1"] - ), - ), - "test.dbt_resto.relationships_table4": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('table-m2')"} - ), - meta={"relationship_type": "one-to-one"}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table-m2", "model.dbt_resto.table-m1"] - ), - ), - "test.dbt_resto.relationships_table1_reverse": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('table-r2')"} - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table-r1", "model.dbt_resto.table-r2"] - ), - ), - "test.dbt_resto.relationships_table1_recursive": ManifestNode( - test_metadata=ManifestNodeTestMetaData( - kwargs={"column_name": "f1", "field": "f2", "to": "ref('table1')"} - ), - meta={}, - columns={}, - depends_on=ManifestNodeDependsOn(nodes=["model.dbt_resto.table1"]), - ), - } - - -@dataclass -class DummyManifestTable: - nodes = { - "model.dbt_resto.table1": ManifestNode( - test_metadata=ManifestNodeTestMetaData(kwargs={}), - meta={}, - raw_sql="--raw_sql--", - database="--database--", - schema_="--schema--", - columns={}, - ), - "model.dbt_resto.table_dummy_columns": ManifestNode( - test_metadata=ManifestNodeTestMetaData(kwargs={}), - meta={}, - raw_sql="--raw_sql--", - database="--database--", - schema_="--schema--", - columns={}, - ), - "model.dbt_resto.table2": ManifestNode( - test_metadata=ManifestNodeTestMetaData(kwargs={}), - meta={}, - raw_sql="--raw_sql2--", - database="--database2--", - schema_="--schema2--", - columns={ - "name2": ManifestNodeColumn(name="name2"), - "name3": ManifestNodeColumn(name="name3"), - }, - ), - } - sources = { - "source.dummy.source_table": ManifestNode( - test_metadata=ManifestNodeTestMetaData(kwargs={}), - meta={}, - database="--database--", - schema_="--schema--", - columns={ - "name1": ManifestNodeColumn(name="name1"), - "name2": ManifestNodeColumn(name="name2"), - }, - ), - } - - -@dataclass -class DummyManifestWithExposure: - exposures = { - "exposure.dbt_resto.dummy": ManifestExposureNode( - depends_on=ManifestNodeDependsOn( - nodes=["model.dbt_resto.table1", "model.dbt_resto.table2"] - ), - ) - } - - -@dataclass -class CatalogNode: - columns: dict - - -@dataclass -class CatalogNodeColumn: - type: str - comment: str = "" - - -@dataclass -class DummyCatalogTable: - nodes = { - "model.dbt_resto.table1": CatalogNode( - columns={"name1": CatalogNodeColumn(type="--name1-type--")} - ), - "model.dbt_resto.table2": CatalogNode( - columns={"name3": CatalogNodeColumn(type="--name3-type--")} - ), - } - sources = { - "source.dummy.source_table": CatalogNode( - columns={"name1": CatalogNodeColumn(type="--name1-type--")} - ), - } +from tests.unit.adapters.algos import ( + DummyCatalogTable, + DummyManifestError, + DummyManifestHasColumns, + DummyManifestRel, + DummyManifestTable, + DummyManifestV6, + DummyManifestV7, + DummyManifestWithExposure, +) class TestAlgoTestRelationship: