Merge pull request #2 from carte-data/flatten-command

Add flatten command

carte_cli/main.py

@@ -12,6 +12,7 @@
 from carte_cli.loader.carte_loader import CarteLoader
 from carte_cli.utils.config_parser import parse_config
 from carte_cli.scaffolding.frontend import create_frontend_dir
+from carte_cli.utils.flatten import flatten as execute_flatten
 
 app = typer.Typer()
 
@@ -58,5 +59,19 @@ def new_frontend(
     create_frontend_dir(name, init_admin=(not no_admin), sample_data=(not no_sample))
 
 
+@app.command("flatten")
+def flatten(
+    input_dir: str = typer.Argument(
+        ..., help="The source metadata directory, the same as the extraction output"
+    ),
+    output_dir: str = typer.Argument(
+        ..., help="The destination directory for flattened markdown files"
+    ),
+    template: str = typer.Option(None, "--template", "-t", help="The template to use for flattening datasets")
+):
+    execute_flatten(input_dir, output_dir, template)
+
+
+
 if __name__ == "__main__":
     app()

carte_cli/utils/flatten.py

@@ -0,0 +1,51 @@
+import os
+import glob
+from typing import Tuple
+from jinja2 import Environment, PackageLoader, select_autoescape, Template
+from pathlib import Path
+
+from jinja2.loaders import FileSystemLoader
+
+import carte_cli.utils.frontmatter as frontmatter
+from carte_cli.model.carte_table_model import TableMetadata
+
+
+def _get_flattened_template(template_path: str):
+    if template_path is None:
+        env = Environment(
+            loader=PackageLoader("carte_cli.utils", "templates"),
+            autoescape=select_autoescape(["md"]),
+        )
+        template = env.get_template("dataset_flattened.md")
+
+    else:
+        env = Environment(loader=FileSystemLoader(searchpath="./"))
+        template = env.get_template(template_path)
+
+    return template
+
+
+def flatten(input_dir: str, output_dir: str, template_path: str) -> None:
+
+    template = _get_flattened_template(template_path)
+
+    file_paths = glob.glob(input_dir + "/*/*/*.md", recursive=True)
+    output_paths = [
+        os.path.join(output_dir, file_path[(len(input_dir)+1) :])
+        for file_path in file_paths
+    ]
+
+    for file_path, output_path in zip(file_paths, output_paths):
+        metadata, content = frontmatter.parse(file_path)
+        dataset = TableMetadata.from_frontmatter(metadata, content)
+        flattened_metadata, flattened_content = flatten_dataset(dataset, template)
+        Path("/".join(output_path.split("/")[:-1])).mkdir(parents=True, exist_ok=True)
+        frontmatter.dump(output_path, flattened_metadata, flattened_content)
+
+    print("Done!")
+
+
+def flatten_dataset(dataset: TableMetadata, template: Template) -> Tuple[dict, str]:
+    metadata = {"title": dataset.name}
+    content = template.render(dataset=dataset)
+    return metadata, content

carte_cli/utils/templates/dataset_flattened.md

@@ -0,0 +1,47 @@
+# {{ dataset.name }} {: .page-title }
+
+<a class="edit-link" href="/admin/#/collections/datasets/entries/{{dataset.connection}}/{{dataset.database}}/{{dataset.name}}"></a>
+
+#### Type: {: .attribute }
+`{{ dataset.table_type.value }}`
+
+<br>
+{: .attribute-break}
+
+#### Database: {: .attribute }
+`{{ dataset.database }}`
+
+<br>
+{: .attribute-break}
+
+#### Location: {: .attribute }
+`{{ dataset.location }}`
+
+<br>
+{: .attribute-break}
+
+## Description
+
+{% if dataset.description.strip() != '' %}
+{{ dataset.description.strip() }}
+{% else %}
+No description
+{: .no-description }
+{% endif %}
+
+## Columns
+
+{% for column in dataset.columns %}
+##### {{ column.name }} {: .column-name }
+
+`{{ column.column_type }}`{: .column-type }
+{% if column.description != none %}
+{{- column.description -}}
+{: .column-description }
+
+{% else %}
+No description
+{: .column-description .no-description }
+
+{% endif %}
+{% endfor %}

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "carte-cli"
-version = "0.2.5"
+version = "0.3.0"
 description = "A static site generator for data catalogs"
 authors = ["Balint Haller <balint@hey.com>"]
 license = "Apache-2.0"
@@ -21,6 +21,7 @@ typer = "^0.3.2"
 click-spinner = "^0.1.10"
 psycopg2 = { version = "^2.8.6", optional = true }
 SQLAlchemy = { version = "^1.3.23", optional = true }
+Jinja2 = "^2.11.3"
 
 [tool.poetry.extras]
 postgres = ["psycopg2", "SQLAlchemy"]