diff --git a/dbterd/adapters/base.py b/dbterd/adapters/base.py
index 7b12171..d7c17aa 100644
--- a/dbterd/adapters/base.py
+++ b/dbterd/adapters/base.py
@@ -22,6 +22,7 @@ def __init__(self, ctx) -> None:
self.ctx = ctx
self.filename_manifest = "manifest.json"
self.filename_catalog = "catalog.json"
+ self.dbt: DbtInvocation = None
def run(self, **kwargs):
"""Main function helps to run by the target strategy"""
@@ -31,6 +32,9 @@ def run(self, **kwargs):
def evaluate_kwargs(self, **kwargs) -> dict:
"""Re-calculate the options
+ - trigger `dbt ls` for re-calculate the Selection if `--dbt` enabled
+ - trigger `dbt docs generate` for re-calculate the artifact direction if `--dbt-atu-artifacts` enabled
+
Raises:
click.UsageError: Not Supported exception
@@ -38,14 +42,21 @@ def evaluate_kwargs(self, **kwargs) -> dict:
dict: kwargs dict
"""
artifacts_dir, dbt_project_dir = self.__get_dir(**kwargs)
- logger.info(f"Using dbt artifact dir at: {artifacts_dir}")
- logger.info(f"Using dbt project dir at: {dbt_project_dir}")
+ logger.info(f"Using dbt project dir at: {dbt_project_dir}")
select = list(kwargs.get("select")) or []
exclude = list(kwargs.get("exclude")) or []
if kwargs.get("dbt"):
+ self.dbt = DbtInvocation(
+ dbt_project_dir=kwargs.get("dbt_project_dir"),
+ dbt_target=kwargs.get("dbt_target"),
+ )
select = self.__get_selection(**kwargs)
exclude = []
+
+ if kwargs.get("dbt_auto_artifacts"):
+ self.dbt.get_artifacts_for_erd()
+ artifacts_dir = f"{dbt_project_dir}/target"
else:
unsupported, rule = has_unsupported_rule(
rules=select.extend(exclude) if exclude else select
@@ -55,6 +66,7 @@ def evaluate_kwargs(self, **kwargs) -> dict:
logger.error(message)
raise click.UsageError(message)
+ logger.info(f"Using dbt artifact dir at: {artifacts_dir}")
kwargs["artifacts_dir"] = artifacts_dir
kwargs["dbt_project_dir"] = dbt_project_dir
kwargs["select"] = select
@@ -91,10 +103,10 @@ def __get_dir(self, **kwargs) -> str:
def __get_selection(self, **kwargs):
"""Override the Selection using dbt's one with `--dbt`"""
- return DbtInvocation(
- dbt_project_dir=kwargs.get("dbt_project_dir"),
- dbt_target=kwargs.get("dbt_target"),
- ).get_selection(
+ if not self.dbt:
+ raise click.UsageError("Flag `--dbt` need to be enabled")
+
+ return self.dbt.get_selection(
select_rules=kwargs.get("select"),
exclude_rules=kwargs.get("exclude"),
)
diff --git a/dbterd/adapters/dbt_invocation.py b/dbterd/adapters/dbt_invocation.py
index fb90dd1..ccf73b1 100644
--- a/dbterd/adapters/dbt_invocation.py
+++ b/dbterd/adapters/dbt_invocation.py
@@ -26,8 +26,52 @@ def __init__(self, dbt_project_dir: str = None, dbt_target: str = None) -> None:
dbt_project_dir or os.environ.get("DBT_PROJECT_DIR") or str(Path.cwd())
)
self.target = dbt_target
+ self.args = ["--quiet", "--log-level", "none"]
+
+ def __invoke(self, runner_args: List[str] = []):
+ """Base function of the dbt invocation
+
+ Args:
+ runner_args (List[str], optional): Actual dbt arguments. Defaults to [].
+
+ Raises:
+ click.UsageError: Invocation failed for a reason
+
+ Returns:
+ dbtRunnerResult.result: dbtRunnerResult.result
+ """
+ args = self.__construct_arguments(*runner_args)
+ logger.debug(f"Invoking: `dbt {' '.join(args)}` at {self.project_dir}")
+ r: dbtRunnerResult = self.dbt.invoke(args)
+
+ if not r.success:
+ logger.error(str(r))
+ raise click.UsageError(str(r))
+
+ return r.result
+
+ def __construct_arguments(self, *args) -> List[str]:
+ """Enrich the dbt arguements with the based options
+
+ Returns:
+ List[str]: Actual dbt arguments
+ """
+ evaluated_args = args
+ if self.args:
+ evaluated_args = [*self.args, *args]
+ if self.project_dir:
+ evaluated_args.extend(["--project-dir", self.project_dir])
+ if self.target:
+ evaluated_args.extend(["--target", self.target])
+
+ return evaluated_args
def __ensure_dbt_installed(self):
+ """Verify if dbt get installed
+
+ Raises:
+ click.UsageError: dbt is not installed
+ """
dbt_spec = importlib.util.find_spec("dbt")
if dbt_spec and dbt_spec.loader:
installed_path = dbt_spec.submodule_search_locations[0]
@@ -55,22 +99,22 @@ def get_selection(
Returns:
List[str]: Selected node names with 'exact' rule
"""
- args = ["ls", "--project-dir", self.project_dir, "--resource-type", "model"]
+ args = ["ls", "--resource-type", "model"]
if select_rules:
args.extend(["--select", " ".join(select_rules)])
if exclude_rules:
args.extend(["--exclude", " ".join(exclude_rules)])
- if self.target:
- args.extend(["--target", self.target])
-
- logger.info(f"Invoking: `dbt {' '.join(args)}` at {self.project_dir}")
- r: dbtRunnerResult = self.dbt.invoke(args)
-
- if not r.success:
- logger.error(str(r))
- raise click.UsageError("str(r)")
+ result = self.__invoke(runner_args=args)
return [
f"exact:model.{str(x).split('.')[0]}.{str(x).split('.')[-1]}"
- for x in r.result
+ for x in result
]
+
+ def get_artifacts_for_erd(self):
+ """Generate dbt artifacts using `dbt docs generate` command
+
+ Returns:
+ dbtRunnerResult: dbtRunnerResult
+ """
+ return self.__invoke(runner_args=["docs", "generate"])
diff --git a/dbterd/cli/params.py b/dbterd/cli/params.py
index 45f8cfc..c82a2d0 100644
--- a/dbterd/cli/params.py
+++ b/dbterd/cli/params.py
@@ -97,6 +97,13 @@ def common_params(func):
default=None,
type=click.STRING,
)
+ @click.option(
+ "--dbt-auto-artifacts",
+ help="Flag to force generating dbt artifact files leveraging Programmatic Invocation",
+ is_flag=True,
+ default=False,
+ show_default=True,
+ )
@functools.wraps(func)
def wrapper(*args, **kwargs):
return func(*args, **kwargs) # pragma: no cover
diff --git a/docs/nav/guide/cli-references.md b/docs/nav/guide/cli-references.md
index b4f1746..40b62a5 100644
--- a/docs/nav/guide/cli-references.md
+++ b/docs/nav/guide/cli-references.md
@@ -18,7 +18,7 @@ run Run the convert
-## run
+## dbterd run
Command to generate diagram-as-a-code file
@@ -58,44 +58,7 @@ Command to generate diagram-as-a-code file
-h, --help Show this message and exit.
```
-### --artifacts-dir (-ad)
-
-Configure the path to directory containing dbt artifact files.
-
-It will take the the nested `/target` directory of `--dbt-project-dir` if not specified.
-
-> Default to the current directory's `/target` if both this option and `--dbt-project-dir` option are not specified
-
-**Examples:**
-=== "CLI"
-
- ```bash
- dbterd run -ad "./target"
- ```
-=== "CLI (long style)"
-
- ```bash
- dbterd run --artifacts-dir "./target"
- ```
-
-### --output (-o)
-
-Configure the path to directory containing the output diagram file.
-> Default to `./target`
-
-**Examples:**
-=== "CLI"
-
- ```bash
- dbterd run -o "./target"
- ```
-=== "CLI (long style)"
-
- ```bash
- dbterd run --output "./target"
- ```
-
-### --select (-s)
+### dbterd run --select (-s)
Selection criteria.
> Select all dbt models if not specified, supports mulitple options
@@ -103,6 +66,7 @@ Selection criteria.
Rules:
- By `name`: model name starts with input string
+- By `exact`: exact model name, formed as `exact:model.package.name`
- By `schema`: schema name starts with an input string, formed as `schema:`
- By `wildcard`: model name matches to a [wildcard pattern](https://docs.python.org/3/library/fnmatch.html), formed as `wildcard:`
- By `exposure`: exposure name, exact match
@@ -111,7 +75,12 @@ Rules:
=== "CLI (by name)"
```bash
- dbterd run -s "model.package_name.model_partital_name"
+ dbterd run -s "model.package.partital_name"
+ ```
+=== "CLI (by exact name)"
+
+ ```bash
+ dbterd run -s "exact:model.package.name"
```
=== "CLI (by schema)"
@@ -121,12 +90,12 @@ Rules:
=== "CLI (by wildcard)"
```bash
- dbterd run --select "wildcard:*xyz"
+ dbterd run -s "wildcard:*xyz"
```
=== "CLI (by exposure)"
```bash
- dbterd run --select "exposure:my_exposure_name"
+ dbterd run -s "exposure:my_exposure_name"
```
#### `AND` and `OR` logic
@@ -148,7 +117,7 @@ Rules:
dbterd run -s schema:abc -s wildcard:*xyz.*
```
-### --exclude (-ns)
+### dbterd run --exclude (-ns)
Exclusion criteria. Rules are the same as Selection Criteria.
> Do not exclude any dbt models if not specified, supports mulitple options
@@ -165,7 +134,44 @@ Exclusion criteria. Rules are the same as Selection Criteria.
dbterd run --exclude 'model.package_name.table'
```
-### --target (-t)
+### dbterd run --artifacts-dir (-ad)
+
+Configure the path to directory containing dbt artifact files.
+
+It will take the the nested `/target` directory of `--dbt-project-dir` if not specified.
+
+> Default to the current directory's `/target` if both this option and `--dbt-project-dir` option are not specified
+
+**Examples:**
+=== "CLI"
+
+ ```bash
+ dbterd run -ad "./target"
+ ```
+=== "CLI (long style)"
+
+ ```bash
+ dbterd run --artifacts-dir "./target"
+ ```
+
+### dbterd run --output (-o)
+
+Configure the path to directory containing the output diagram file.
+> Default to `./target`
+
+**Examples:**
+=== "CLI"
+
+ ```bash
+ dbterd run -o "./target"
+ ```
+=== "CLI (long style)"
+
+ ```bash
+ dbterd run --output "./target"
+ ```
+
+### dbterd run --target (-t)
Target to the diagram-as-code platform
> Default to `dbml`
@@ -184,7 +190,7 @@ Supported target, please visit [Generate the Targets](https://dbterd.datnguyen.d
dbterd run --target dbml
```
-### --algo (-a)
+### dbterd run --algo (-a)
Specified algorithm in the way to detect diagram connectors
> Default to `test_relationship`
@@ -226,7 +232,7 @@ In the above:
dbterd run --algo "test_relationship:(name:foreign_key|c_from:fk_column_name|c_to:pk_column_name)"
```
-### --manifest-version (-mv)
+### dbterd run --manifest-version (-mv)
Specified dbt manifest.json version
> Auto detect if not specified
@@ -243,7 +249,7 @@ Specified dbt manifest.json version
dbterd run -mv 7
```
-### --catalog-version (-cv)
+### dbterd run --catalog-version (-cv)
Specified dbt catalog.json version
> Auto detect if not specified
@@ -260,7 +266,7 @@ Specified dbt catalog.json version
dbterd run -cv 7
```
-### --resource-type (-rt)
+### dbterd run --resource-type (-rt)
Specified dbt resource type(seed, model, source, snapshot).
> Default to `["model"]`, supports mulitple options
@@ -277,7 +283,7 @@ Specified dbt resource type(seed, model, source, snapshot).
dbterd run --resource-type model
```
-### --dbt
+### dbterd run --dbt
Flag to indicate the Selecton to follow dbt's one leveraging Programmatic Invocation
> Default to `False`
@@ -296,7 +302,22 @@ Flag to indicate the Selecton to follow dbt's one leveraging Programmatic Invoca
# select starts with 'something'
```
-### --dbt-project-dir (-dpd)
+### dbterd run --dbt --dbt-auto-artifact
+
+Flag to indicate force running `dbt docs generate` to the targetted project in order to produce the dbt artifact files.
+
+This option have to be enabled together with `--dbt` option, and will override the value of `--artifacts-dir` to be using the `/target` dir of the value of `--dbt-project-dir`.
+
+> Default to `False`
+
+**Examples:**
+=== "CLI"
+
+ ```bash
+ dbterd run -s +something --dbt --dbt-auto-artifacts
+ ```
+
+### dbterd run --dbt-project-dir (-dpd)
Specified dbt project directory path
@@ -313,7 +334,7 @@ You should specified this option if your CWD is not the dbt project dir, and nor
# the artifacts dir will probably be assumed as: /path/to/dbt/project/target
```
-### --dbt-target (-dt)
+### dbterd run --dbt-target (-dt)
Specified dbt target name
@@ -330,9 +351,9 @@ Probably used with `--dbt` enabled.
# the artifacts dir will probably be assumed as: /path/to/dbt/project/target
```
-## debug
+## dbterd debug
-Shows hidden configured values
+Shows hidden configured values, which will help us to see what configs are passed into and how they are evaluated to be used.
**Examples:**
=== "Output"
diff --git a/tests/unit/adapters/test_base.py b/tests/unit/adapters/test_base.py
index 6b96051..178095b 100644
--- a/tests/unit/adapters/test_base.py
+++ b/tests/unit/adapters/test_base.py
@@ -6,6 +6,7 @@
from dbterd import default
from dbterd.adapters.base import Executor
+from dbterd.adapters.dbt_invocation import DbtInvocation
class TestBase:
@@ -50,11 +51,20 @@ def test___read_catalog(self):
)
def test__get_selection(self, mock_dbt_invocation):
worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy")))
+ worker.dbt = DbtInvocation()
assert "dummy" == worker._Executor__get_selection(
select_rules=[], exclude_rules=[]
)
mock_dbt_invocation.assert_called_once()
+ @mock.patch(
+ "dbterd.adapters.base.DbtInvocation.get_selection", return_value="dummy"
+ )
+ def test__get_selection__error(self, mock_dbt_invocation):
+ worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy")))
+ with pytest.raises(click.UsageError):
+ worker._Executor__get_selection()
+
@pytest.mark.parametrize(
"kwargs, expected",
[
@@ -80,16 +90,37 @@ def test__get_selection(self, mock_dbt_invocation):
exclude=[],
),
),
+ (
+ dict(select=[], exclude=[], dbt=True, dbt_auto_artifacts=True),
+ dict(
+ dbt=True,
+ dbt_auto_artifacts=True,
+ artifacts_dir="/path/dpd/target",
+ dbt_project_dir="/path/dpd",
+ select=["yolo"],
+ exclude=[],
+ ),
+ ),
],
)
@mock.patch("dbterd.adapters.base.Executor._Executor__get_dir")
@mock.patch("dbterd.adapters.base.Executor._Executor__get_selection")
- def test_evaluate_kwargs(self, mock_get_selection, mock_get_dir, kwargs, expected):
+ @mock.patch("dbterd.adapters.base.DbtInvocation.get_artifacts_for_erd")
+ def test_evaluate_kwargs(
+ self,
+ mock_get_artifacts_for_erd,
+ mock_get_selection,
+ mock_get_dir,
+ kwargs,
+ expected,
+ ):
worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy")))
mock_get_dir.return_value = ("/path/ad", "/path/dpd")
mock_get_selection.return_value = ["yolo"]
assert expected == worker.evaluate_kwargs(**kwargs)
mock_get_dir.assert_called_once()
+ if kwargs.get("dbt_auto_artifacts"):
+ mock_get_artifacts_for_erd.assert_called_once()
@pytest.mark.parametrize(
"kwargs, mock_isfile_se, expected",
diff --git a/tests/unit/adapters/test_dbt_invocation.py b/tests/unit/adapters/test_dbt_invocation.py
index 5fdf354..0b3f814 100644
--- a/tests/unit/adapters/test_dbt_invocation.py
+++ b/tests/unit/adapters/test_dbt_invocation.py
@@ -49,12 +49,18 @@ def test_get_selection(
)
assert actual == expected
- args = ["ls", "--project-dir", invoker.project_dir, "--resource-type", "model"]
+ args = ["ls", "--resource-type", "model"]
if select_rules:
args.extend(["--select", " ".join(select_rules)])
if exclude_rules:
args.extend(["--exclude", " ".join(exclude_rules)])
- mock_dbtRunner_invoke.assert_called_once_with(args)
+ mock_dbtRunner_invoke.assert_called_once_with(
+ [
+ *["--quiet", "--log-level", "none"],
+ *args,
+ *["--project-dir", invoker.project_dir],
+ ]
+ )
@mock.patch("dbterd.adapters.dbt_invocation.dbtRunner.invoke")
def test_get_selection__failed(self, mock_dbtRunner_invoke):
@@ -63,3 +69,17 @@ def test_get_selection__failed(self, mock_dbtRunner_invoke):
DbtInvocation(dbt_target="dummy").get_selection(
select_rules=[], exclude_rules=[]
)
+
+ @mock.patch("dbterd.adapters.dbt_invocation.dbtRunner.invoke")
+ def test_get_artifacts_for_erd(self, mock_dbtRunner_invoke):
+ invoker = DbtInvocation()
+ _ = invoker.get_artifacts_for_erd()
+
+ args = ["docs", "generate"]
+ mock_dbtRunner_invoke.assert_called_once_with(
+ [
+ *["--quiet", "--log-level", "none"],
+ *args,
+ *["--project-dir", invoker.project_dir],
+ ]
+ )