Merge pull request #64 from ikanashov/MG-3809

MG-3809 harmonize with new version of DD-API and DD-UI

Author: ikanashov

data-detective-etl/common/builders/__init__.py

@@ -1,7 +1,4 @@
-from common.builders.table_info_builder import (
-    TableInfoDescriptionType,
-    TableInfoBuilder
-)
+from common.builders.table_info_builder import TableInfoDescriptionType, TableInfoBuilder
 from common.builders.html_info_builder import HtmlInfoBuilder
 from common.builders.code_builder import CodeBuilder
 from common.builders.json_system_builder import JsonSystemBuilder

data-detective-etl/common/builders/code_builder.py

@@ -5,15 +5,21 @@
 
 class CodeBuilder:
 
-    __slots__ = ('_header', '_language', '_key', '_data', '_opened',)
+    __slots__ = (
+        '_header',
+        '_language',
+        '_key',
+        '_data',
+        '_opened',
+    )
 
     def __init__(
-            self,
-            header: str = None,
-            language: str = None,
-            key: List[str] = None,
-            data: str = None,
-            opened: bool = True,
+        self,
+        header: str = None,
+        language: str = None,
+        key: List[str] = None,
+        data: str = None,
+        opened: bool = True,
     ):
         self._header = header
         self._language = language
@@ -22,13 +28,13 @@ def __init__(
         self._opened = opened
 
     def __call__(
-            self,
-            header: str = None,
-            language: str = None,
-            key: List[str] = None,
-            data: str = None,
-            opened: bool = None,
-            **kwargs
+        self,
+        header: str = None,
+        language: str = None,
+        key: List[str] = None,
+        data: str = None,
+        opened: bool = None,
+        **kwargs,
     ) -> _CodeInfoResultType:
         if opened is None:
             opened = self._opened

data-detective-etl/common/builders/html_info_builder.py

@@ -9,14 +9,18 @@ class HtmlInfoBuilder:
     """Build html_info by source data
     build block with data field and container contents
     """
+
     __slots__ = ('_header_mapping', '_type', '_opened')
 
-    def __init__(self, header_mapping: Mapping[Text, Text],
-                 opened: bool = None):
+    def __init__(self, header_mapping: Mapping[Text, Text], opened: bool = None):
         self._header_mapping = header_mapping
         self._opened = opened
 
     def __call__(self, row_data) -> _HtmlInfoResultType:
-        return [{'data': row_data[k], 'header': v, 'opened': str(int(self._opened))} if isinstance(self._opened, bool)
-                else {'data': row_data[k], 'header': v} for k, v in self._header_mapping.items()
-                if isnotempty(row_data.get(k))]
+        return [
+            {'data': row_data[k], 'header': v, 'opened': str(int(self._opened))}
+            if isinstance(self._opened, bool)
+            else {'data': row_data[k], 'header': v}
+            for k, v in self._header_mapping.items()
+            if isnotempty(row_data.get(k))
+        ]

data-detective-etl/common/builders/json_system_builder.py

@@ -8,24 +8,24 @@ class JsonSystemBuilder:
     __slots__ = ('_system_for_search', '_type_for_search', '_card_type', '_business_type')
 
     def __init__(
-            self,
-            system_for_search: str = None,
-            type_for_search: str = None,
-            card_type: str = None,
-            business_type: str = None,
+        self,
+        system_for_search: str = None,
+        type_for_search: str = None,
+        card_type: str = None,
+        business_type: str = None,
     ):
         self._system_for_search = system_for_search
         self._type_for_search = type_for_search
         self._card_type = card_type
         self._business_type = business_type
 
     def __call__(
-            self,
-            system_for_search: str = None,
-            type_for_search: str = None,
-            card_type: str = None,
-            business_type: str = None,
-            **kwargs
+        self,
+        system_for_search: str = None,
+        type_for_search: str = None,
+        card_type: str = None,
+        business_type: str = None,
+        **kwargs,
     ) -> _JsonSystemResultType:
         result = dict(
             system_for_search=system_for_search or self._system_for_search,

data-detective-etl/common/builders/link_builder.py

@@ -13,6 +13,7 @@ def __init__(self, link_types: Mapping[Text, Text] = None):
         self._link_types = link_types or self._map
 
     def __call__(self, row_data) -> _LinkResultType:
-        result = [{'link': row_data.pop(k), 'type': v}
-                  for k, v in self._link_types.items() if isnotempty(row_data.get(k))]
+        result = [
+            {'link': row_data.pop(k), 'type': v} for k, v in self._link_types.items() if isnotempty(row_data.get(k))
+        ]
         return result or None

data-detective-etl/common/builders/table_info_builder.py

@@ -5,13 +5,9 @@
 from common.utils import isnotempty
 
 
-_TableInfoResultType = Dict[str, Union[
-    Optional[str],
-    List[str],
-    List[Optional[str]],
-    List[Dict[str, str]],
-    List[Dict[str, Optional[str]]]
-]]
+_TableInfoResultType = Dict[
+    str, Union[Optional[str], List[str], List[Optional[str]], List[Dict[str, str]], List[Dict[str, Optional[str]]]]
+]
 
 
 class TableInfoDescriptionType(BaseModel):
@@ -31,22 +27,27 @@ def __init__(self, table: TableInfoDescriptionType):
 
     def _apply_serializers(self, row_data: Dict) -> Dict:
         serializers = self._table.serializers or dict()
-        return {value: serializers.get(key, str)(row_data[key])
-                for key, value in self._table.keys.items() if isnotempty(row_data.get(key))}
+        return {
+            value: serializers.get(key, str)(row_data[key])
+            for key, value in self._table.keys.items()
+            if isnotempty(row_data.get(key))
+        }
 
     @staticmethod
     def _get_all_keys(row_data):
-        return {k for row in row_data for k in row.keys()} \
-            if isinstance(row_data, (list, tuple)) else row_data.keys()
+        return {k for row in row_data for k in row.keys()} if isinstance(row_data, (list, tuple)) else row_data.keys()
 
     def _get_kv_table(self, row_data) -> _TableInfoResultType:
         serializers: Dict[Text, Callable[..., Text]] = self._table.serializers or dict()
         return dict(
             columns=['Key', 'Value'],
-            data=[{'Key': value, 'Value': serializers.get(key, str)(row_data[key])}
-                  for key, value in self._table.keys.items() if isnotempty(row_data.get(key))],
+            data=[
+                {'Key': value, 'Value': serializers.get(key, str)(row_data[key])}
+                for key, value in self._table.keys.items()
+                if isnotempty(row_data.get(key))
+            ],
             header=self._table.header,
-            display_headers=self._table.display_headers
+            display_headers=self._table.display_headers,
         )
 
     def _get_table(self, row_data) -> _TableInfoResultType:
@@ -55,10 +56,11 @@ def _get_table(self, row_data) -> _TableInfoResultType:
         columns.sort()
         return dict(
             columns=columns,
-            data=[self._apply_serializers(row) for row in row_data] if isinstance(row_data, list)
+            data=[self._apply_serializers(row) for row in row_data]
+            if isinstance(row_data, list)
             else [self._apply_serializers(row_data)],
             header=self._table.header,
-            display_headers=self._table.display_headers
+            display_headers=self._table.display_headers,
         )
 
     def get_table(self, row_data) -> _TableInfoResultType:

data-detective-etl/common/root_tree_generator.py

@@ -36,9 +36,11 @@ def print_table(table):
         print('```')
         bfs(graph_yaml, 'root', root_table=root)
         print('```')
-        print('''
+        print(
+            '''
 Description of hierarchy of root entities:
 
 | Name | Path to entity | Entity type | Info |
-| --- | --- | --- | --- |''')
+| --- | --- | --- | --- |'''
+        )
         print_table(root)

data-detective-etl/common/utilities/entity_enums.py

@@ -1,16 +1,18 @@
+from collections import namedtuple
+from enum import Enum
 from typing import FrozenSet
 
 
-class EntityTypes:
-    TREE_NODE = 'TREE_NODE'
-    SCHEMA = 'SCHEMA'
-    TABLE = 'TABLE'
-    COLUMN = 'COLUMN'
-    JOB = 'JOB'
-    LOGICAL_SCHEMA = 'LOGICAL_SCHEMA'
-    LOGICAL_TABLE = 'LOGICAL_TABLE'
-    LOGICAL_COLUMN = 'LOGICAL_COLUMN'
-    LOGICAL_REPORT = 'LOGICAL_REPORT'
+class EntityTypes(namedtuple('EntityType', 'key message_code'), Enum):
+    TREE_NODE = 'TREE_NODE', 'data.entity.entity-type.tree-node'
+    SCHEMA = 'SCHEMA', 'data.entity.entity-type.schema'
+    TABLE = 'TABLE', 'data.entity.entity-type.table'
+    COLUMN = 'COLUMN', 'data.entity.entity-type.column'
+    JOB = 'JOB', 'data.entity.entity-type.job'
+    LOGICAL_SCHEMA = 'LOGICAL_SCHEMA', 'data.entity.entity-type.logical-schema'
+    LOGICAL_TABLE = 'LOGICAL_TABLE', 'data.entity.entity-type.logical-table'
+    LOGICAL_COLUMN = 'LOGICAL_COLUMN', 'data.entity.entity-type.logical-column'
+    LOGICAL_REPORT = 'LOGICAL_REPORT', 'data.entity.entity-type.logical-report'
 
 
 class EntityFields:
@@ -45,8 +47,16 @@ class RelationFields:
     ATTRIBUTE = 'attribute'
 
 
-ENTITY_CORE_FIELDS: FrozenSet[str] = frozenset({EntityFields.URN, EntityFields.ENTITY_TYPE,
-                                                EntityFields.ENTITY_NAME, EntityFields.ENTITY_NAME_SHORT,
-                                                EntityFields.JSON_DATA, EntityFields.SEARCH_DATA})
-RELATION_CORE_FIELDS: FrozenSet[str] = frozenset({RelationFields.SOURCE, RelationFields.DESTINATION,
-                                                  RelationFields.TYPE, RelationFields.ATTRIBUTE})
+ENTITY_CORE_FIELDS: FrozenSet[str] = frozenset(
+    {
+        EntityFields.URN,
+        EntityFields.ENTITY_TYPE,
+        EntityFields.ENTITY_NAME,
+        EntityFields.ENTITY_NAME_SHORT,
+        EntityFields.JSON_DATA,
+        EntityFields.SEARCH_DATA,
+    }
+)
+RELATION_CORE_FIELDS: FrozenSet[str] = frozenset(
+    {RelationFields.SOURCE, RelationFields.DESTINATION, RelationFields.TYPE, RelationFields.ATTRIBUTE}
+)

data-detective-etl/common/utilities/search_enums.py

@@ -1,30 +1,51 @@
 from collections import namedtuple
 from enum import Enum
+from typing import Union
 
 SYSTEM_FOR_SEARCH = 'system_for_search'
 TYPE_FOR_SEARCH = 'type_for_search'
 
 
 # https://stackoverflow.com/a/62601113/4545870
-class SystemForSearch(namedtuple('SearchSystem', 'name description'), Enum):
-    ORACLE = 'Oracle', 'Oracle databases from replication'
-    POSTGRES = 'Postgres', 'Postgres databases from replication'
-    LOGICAL_MODEL = 'Logical Model', 'Logical Model DWH'
-    DATA_DETECTIVE = 'Data Detective', 'Data Catalog for entities of any type'
+class SystemForSearch(namedtuple('SearchSystem', 'name title_code info_code'), Enum):
+    ORACLE = 'Oracle', 'data.search.filters.system.oracle.title', 'data.search.filters.system.oracle.info'
+    POSTGRES = 'Postgres', 'data.search.filters.system.postgres.title', 'data.search.filters.system.postgres.info'
+    LOGICAL_MODEL = (
+        'Logical_Model',
+        'data.search.filters.system.logical-model.title',
+        'data.search.filters.system.logical-model.info',
+    )
+    DATA_DETECTIVE = (
+        'Data Detective',
+        'data.search.filters.system.data-detective.title',
+        'data.search.filters.system.data-detective.info',
+    )
 
     def __str__(self) -> str:
         return self.name
 
 
-class TypeForSearch(namedtuple('SearchType', 'name description'), Enum):
-    COLUMN = 'Column', 'Physical column of physical table'
-    TABLE = 'Table', 'Physical table or view in database'
-    SCHEMA = 'Schema', 'Physical schema in database'
-    DATABASE = 'Database', 'Database from DBMS'
-    JOB = 'Job', 'ETL job, pipeline, DAG or workflow'
-    LOGICAL_SCHEMA = 'Logical Schema', 'Logical schema in model'
-    LOGICAL_TABLE = 'Logical Table', 'Logical table in model'
-    LOGICAL_COLUMN = 'Logical Column', 'Column in logical table'
+class TypeForSearch(namedtuple('SearchType', 'name title_code info_code'), Enum):
+    COLUMN = 'Column', 'data.search.filters.type.column.title', 'data.search.filters.type.column.info'
+    TABLE = 'Table', 'data.search.filters.type.table.title', 'data.search.filters.type.table.info'
+    SCHEMA = 'Schema', 'data.search.filters.type.schema.title', 'data.search.filters.type.schema.info'
+    DATABASE = 'Database', 'data.search.filters.type.database.title', 'data.search.filters.type.database.info'
+    JOB = 'Job', 'data.search.filters.type.job.title', 'data.search.filters.type.job.info'
+    LOGICAL_SCHEMA = (
+        'Logical Schema',
+        'data.search.filters.type.logical-schema.title',
+        'data.search.filters.type.logical-schema.info',
+    )
+    LOGICAL_TABLE = (
+        'Logical Table',
+        'data.search.filters.type.logical-table.title',
+        'data.search.filters.type.logical-table.info',
+    )
+    LOGICAL_COLUMN = (
+        'Logical Column',
+        'data.search.filters.type.logical-column.title',
+        'data.search.filters.type.logical-column.info',
+    )
 
     def __str__(self) -> str:
         return self.name
@@ -38,3 +59,39 @@ class CardType(namedtuple('CardType', 'name description'), Enum):
 
     def __str__(self) -> str:
         return self.name
+
+
+system_for_search_x_type_for_search: list[dict[str, Union[str, list[str]]]] = [
+    {
+        "system_name": SystemForSearch.ORACLE.name,
+        "type_name": [
+            TypeForSearch.COLUMN.name,
+            TypeForSearch.TABLE.name,
+            TypeForSearch.SCHEMA.name,
+            TypeForSearch.DATABASE.name,
+        ],
+    },
+    {
+        "system_name": SystemForSearch.POSTGRES.name,
+        "type_name": [
+            TypeForSearch.COLUMN.name,
+            TypeForSearch.TABLE.name,
+            TypeForSearch.SCHEMA.name,
+            TypeForSearch.DATABASE.name,
+        ],
+    },
+    {
+        "system_name": SystemForSearch.LOGICAL_MODEL.name,
+        "type_name": [
+            TypeForSearch.LOGICAL_COLUMN.name,
+            TypeForSearch.LOGICAL_TABLE.name,
+            TypeForSearch.LOGICAL_SCHEMA.name,
+        ],
+    },
+    {
+        "system_name": SystemForSearch.DATA_DETECTIVE.name,
+        "type_name": [
+            TypeForSearch.JOB.name,
+        ],
+    },
+]