feat(eda.create-db-report): add orphan page and fix foreign key bug

.gitignore

@@ -123,6 +123,7 @@ dataprep/eda/create_db_report/layout/index.html
 dataprep/eda/create_db_report/layout/columns.html
 dataprep/eda/create_db_report/layout/constraints.html
 dataprep/eda/create_db_report/layout/relationships.html
+dataprep/eda/create_db_report/layout/orphans.html
 dataprep/eda/create_db_report/layout/diagrams/*
 dataprep/eda/create_db_report/layout/tables/*
 !dataprep/eda/create_db_report/layout/tables/table.html

dataprep/eda/create_db_report/db_models/table_column.py

@@ -18,7 +18,7 @@ def __init__(
         self.type_name = type_name
         self.not_null = not_null
         self.default_value = default_value.replace("'", "") if default_value else ""
-        self.comments = comments.replace("'", "") if default_value else ""
+        self.comments = comments.replace("'", "") if comments else ""
         self.parents = {}
         self.children = {}
         self.type = 0

dataprep/eda/create_db_report/diagram_factory.py

@@ -62,8 +62,10 @@ def generate_summary_diagram(self, database_object: Database, database_url: str)
         return json_tables, json_relationships
 
     def generate_table_diagrams(self, database_object: Database, database_url: str):
+        # Generate diagram for each table
         database_tables = database_object.get_tables_dict()
         table_names = set(database_tables.keys())
+        orphan_table_names = list()
         result_tables = {}
         for table in table_names:
             related_table_names = {table}
@@ -74,6 +76,8 @@ def generate_table_diagrams(self, database_object: Database, database_url: str):
                     table_foreign_keys[foreign_key].get_parent_table().get_name()
                 )
             related_table_names = list(related_table_names)
+            if len(related_table_names) == 1:
+                orphan_table_names.append(table)
             os.chdir(self.dirs["table"])
             if _WITH_GV:
                 render_er(
@@ -96,7 +100,29 @@ def generate_table_diagrams(self, database_object: Database, database_url: str):
                 "json_tables": json_tables,
                 "json_relationships": json_relationships,
             }
-        return result_tables
+
+        # Generate diagram for orphan tables
+        os.chdir(self.dirs["orphan"])
+        if _WITH_GV:
+            render_er(database_url, "orphans.dot", include_tables=" ".join(orphan_table_names))
+        else:
+            raise self.import_err
+        os.chdir(self.cwd)
+        orphan_tables = {
+            key: value for key, value in database_tables.items() if key in orphan_table_names
+        }
+        json_tables = self.generate_diagram_tables(orphan_tables)
+        file = str(
+            os.path.realpath(
+                os.path.join(os.path.dirname(__file__), "layout/diagrams/orphans/orphans.dot")
+            )
+        )
+        json_relationships = self.generate_diagram_relationships(file)
+        orphan_result_tables = {
+            "json_tables": json_tables,
+            "json_relationships": json_relationships,
+        }
+        return orphan_result_tables, result_tables
 
     def generate_diagram_tables(self, tables: Dict[str, Any]):
         table_names = set(tables.keys())

dataprep/eda/create_db_report/header/sql_metadata.py

@@ -518,8 +518,8 @@ def plot_sqlite_db(sqliteConnection: Engine, analyze: bool = False):
         temp_pk_fk = (
             pk_fk[pk_fk["table_name"] == i].drop(columns=["table_name"]).to_dict(orient="records")
         )
+        fk_counter, uk_counter = 1, 1
         for j in temp_pk_fk:
-            fk_counter, uk_counter = 1, 1
             if j["constraint_type"] == "primary key":
                 element = i + "_pkey"
                 temp["constraints"][element] = {}

dataprep/eda/create_db_report/layout/container.html

@@ -63,7 +63,7 @@
                                 <li><a href="{{ root_path }}columns.html" title="All of the columns in the schema">Columns</a></li>
                                 <li><a href="{{ root_path }}constraints.html" title="Useful for diagnosing error messages that just give constraint name or number">Constraints</a></li>
                                 <li><a href="{{ root_path }}relationships.html" title="Diagram of table relationships">Relationships</a></li>
-<!--                                <li><a href="{{ root_path }}orphans.html" title="View of tables with neither parents nor children">Orphan&nbsp;Tables</a></li>-->
+                                <li><a href="{{ root_path }}orphans.html" title="View of tables with neither parents nor children">Orphan&nbsp;Tables</a></li>
 <!--                                <li><a href="{{ root_path }}anomalies.html" title="Things that might not be quite right">Anomalies</a></li>-->
 <!--                                <li><a href="{{ root_path }}routines.html" title="Procedures and functions">Routines</a></li>-->
                             </ul>

dataprep/eda/create_db_report/layout/diagram.html

@@ -5,19 +5,25 @@
   // For details, see https://gojs.net/latest/intro/buildingObjects.html
   const $ = go.GraphObject.make;  // for conciseness in defining templates
 
+  let layoutSelect = document.getElementById('diagram-layout-list');
+
   myDiagram =
     $(go.Diagram, "myDiagramDiv",  // must name or refer to the DIV HTML element
       {
         allowDelete: false,
         allowCopy: false,
-        layout: $(go.LayeredDigraphLayout),
+        layout: $(go[layoutSelect[0].value]),
         "undoManager.isEnabled": true
       });
     function diagramInfo(model) {
       return "Model:\n" + model.nodeDataArray.length + " nodes, " +
                           model.linkDataArray.length + " links";
     }
 
+  layoutSelect.addEventListener("change", (event) => {
+      myDiagram.layout = $(go[event.target.value]);
+  });
+
   //provide a tooltip for the background of the Diagram, when not over any Part
   myDiagram.toolTip =
   $("ToolTip",
@@ -187,11 +193,6 @@
     });
 
   zoomSlider = new ZoomSlider(myDiagram);
-
-  let layoutSelect = document.getElementById('diagram-layout-list');
-  layoutSelect.addEventListener("change", (event) => {
-      myDiagram.layout = $(go[event.target.value]);
-  });
 }
 window.addEventListener('DOMContentLoaded', init);
 </script>

dataprep/eda/create_db_report/layout/orphan.html

@@ -0,0 +1,46 @@
+<!-- Content Header (Page header) -->
+                <section class="content-header">
+                    <h1>Utility Tables</h1>
+                </section>
+                <!-- Main content -->
+                <section class="content">
+                    {% if diagram_tables %}
+                    <div class="box box-primary">
+                        <div class="box-header with-border">
+                            <i class="fa fa-code-fork"></i>
+                            <h3 class="box-title">Orphan Tables</h3>
+                        </div>
+                        <section class="interactive-diagrams">
+                            <label for="diagram-layout-list">Layout</label>
+                            <select id="diagram-layout-list">
+                                <option value="CircularLayout">Circular</option>
+                                <option value="LayeredDigraphLayout">Layered Digraph</option>
+                                <option value="GridLayout">Grid</option>
+                                <option value="TreeLayout">Tree</option>
+                                <option value="ForceDirectedLayout">Force-Directed</option>
+                            </select>
+                            <div id="interactive-diagrams" class="p-4 w-full">
+                                <div id="myDiagramDiv"
+                                     style="background-color: white; border: 1px solid black; width: 100%; height: 1000px; position: relative; cursor: auto;">
+                                    <canvas tabindex="0" width="2108" height="1396"
+                                            style="position: absolute; top: 0px; left: 0px; z-index: 2; touch-action: none; width: 1054px; height: 698px; cursor: auto;">
+                                        This text is displayed if your browser does not support the Canvas HTML element.
+                                    </canvas>
+                                    <div style="position: absolute; overflow: auto; width: 1054px; height: 698px; z-index: 1;">
+                                        <div style="position: absolute; width: 1px; height: 1px;"></div>
+                                    </div>
+                                </div>
+                            </div>
+                        </section>
+                    </div>
+                    {% endif %}
+                    {% if not diagram_tables %}
+                    <div class="alert alert-warning alert-dismissible">
+                        <button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button>
+                        <h4><i class="icon fa fa-warning"></i>Not all diagrams were created</h4>
+                        <p>SchemaSpy was unable to generate diagrams for all orphan tables.<br />
+                            Check application output for details.
+                        </p>
+                    </div>
+                    {% endif %}
+                </section>

dataprep/eda/create_db_report/layout/relationship.html

@@ -13,14 +13,14 @@ <h3 class="box-title">Diagrams</h3>
         <section class="interactive-diagrams">
             <label for="diagram-layout-list">Layout</label>
             <select id="diagram-layout-list">
-                <option value="LayeredDigraphLayout" >Layered Digraph</option>
                 <option value="GridLayout">Grid</option>
+                <option value="LayeredDigraphLayout" >Layered Digraph</option>
                 <option value="TreeLayout">Tree</option>
                 <option value="ForceDirectedLayout">Force-Directed</option>
                 <option value="CircularLayout">Circular</option>
             </select>
             <div id="interactive-diagrams" class="p-4 w-full">
-                  <div id="myDiagramDiv" style="background-color: white; border: 1px solid black; width: 100%; height: 700px; position: relative; cursor: auto;"><canvas tabindex="0" width="2108" height="1396" style="position: absolute; top: 0px; left: 0px; z-index: 2; touch-action: none; width: 1054px; height: 698px; cursor: auto;">This text is displayed if your browser does not support the Canvas HTML element.</canvas><div style="position: absolute; overflow: auto; width: 1054px; height: 698px; z-index: 1;"><div style="position: absolute; width: 1px; height: 1px;"></div></div></div>
+                  <div id="myDiagramDiv" style="background-color: white; border: 1px solid black; width: 100%; height: 1000px; position: relative; cursor: auto;"><canvas tabindex="0" width="2108" height="1396" style="position: absolute; top: 0px; left: 0px; z-index: 2; touch-action: none; width: 1054px; height: 698px; cursor: auto;">This text is displayed if your browser does not support the Canvas HTML element.</canvas><div style="position: absolute; overflow: auto; width: 1054px; height: 698px; z-index: 1;"><div style="position: absolute; width: 1px; height: 1px;"></div></div></div>
             </div>
         </section>
     </div>

dataprep/eda/create_db_report/layout/tables/table.html

@@ -134,10 +134,10 @@ <h3 class="box-title">Relationships</h3>
                         <section class="interactive-diagrams">
                             <label for="diagram-layout-list">Layout</label>
                             <select id="diagram-layout-list">
+                                <option value="ForceDirectedLayout">Force-Directed</option>
                                 <option value="LayeredDigraphLayout" >Layered Digraph</option>
                                 <option value="GridLayout">Grid</option>
                                 <option value="TreeLayout">Tree</option>
-                                <option value="ForceDirectedLayout">Force-Directed</option>
                                 <option value="CircularLayout">Circular</option>
                             </select>
                             <div id="interactive-diagrams" class="p-4 w-full">

dataprep/eda/create_db_report/run_function.py

@@ -15,6 +15,7 @@
 from .views.constraint import ConstraintPage
 from .views.table import TablePage
 from .views.relationship import RelationshipPage
+from .views.orphan import OrphanPage
 from .diagram_factory import DiagramFactory
 from .header.sql_metadata import plot_mysql_db, plot_postgres_db, plot_sqlite_db
 
@@ -249,8 +250,17 @@ def generate_db_report(sql_engine: Engine, analyze: bool = False):
     )
     constraint_page.page_writer(constraints, current_database.get_tables(), file)
 
+    orphan_result_tables, result_tables = diagram_generator.generate_table_diagrams(
+        current_database, str(sql_engine.url)
+    )
+
+    orphan_page = OrphanPage(template_compiler)
+    file = str(os.path.realpath(os.path.join(os.path.dirname(__file__), "layout", "orphans.html")))
+    orphan_page.page_writer(
+        orphan_result_tables["json_tables"], orphan_result_tables["json_relationships"], file
+    )
+
     table_files = ["table.html", "table.js"]
-    result_tables = diagram_generator.generate_table_diagrams(current_database, str(sql_engine.url))
     for table in current_database.get_tables():
         table_file_name = table.name + ".html"
         table_files.append(table_file_name)

dataprep/eda/create_db_report/views/orphan.py

@@ -0,0 +1,25 @@
+import json
+from typing import List, Any
+from ..page_models.page_data import PageData
+from ..page_models.page_template import PageTemplate
+
+
+class OrphanPage:
+    def __init__(self, template_object: PageTemplate) -> None:
+        self.template_object = template_object
+
+    def page_writer(
+        self,
+        json_tables: List[Any],
+        json_relationships: List[Any],
+        new_file: str,
+    ):
+        """
+        Compile the data needed by the pystache template for orphan page
+        """
+        page_data = PageData("orphan.html", "")
+        page_data.add_scope("diagram_tables", json.dumps(json_tables))
+        page_data.add_scope("diagram_relationships", json.dumps(json_relationships))
+        page_data.set_depth(0)
+
+        return self.template_object.write_data(page_data, new_file, "", {})