diff --git a/dbterd/adapters/base.py b/dbterd/adapters/base.py index 7b12171..d7c17aa 100644 --- a/dbterd/adapters/base.py +++ b/dbterd/adapters/base.py @@ -22,6 +22,7 @@ def __init__(self, ctx) -> None: self.ctx = ctx self.filename_manifest = "manifest.json" self.filename_catalog = "catalog.json" + self.dbt: DbtInvocation = None def run(self, **kwargs): """Main function helps to run by the target strategy""" @@ -31,6 +32,9 @@ def run(self, **kwargs): def evaluate_kwargs(self, **kwargs) -> dict: """Re-calculate the options + - trigger `dbt ls` for re-calculate the Selection if `--dbt` enabled + - trigger `dbt docs generate` for re-calculate the artifact direction if `--dbt-atu-artifacts` enabled + Raises: click.UsageError: Not Supported exception @@ -38,14 +42,21 @@ def evaluate_kwargs(self, **kwargs) -> dict: dict: kwargs dict """ artifacts_dir, dbt_project_dir = self.__get_dir(**kwargs) - logger.info(f"Using dbt artifact dir at: {artifacts_dir}") - logger.info(f"Using dbt project dir at: {dbt_project_dir}") + logger.info(f"Using dbt project dir at: {dbt_project_dir}") select = list(kwargs.get("select")) or [] exclude = list(kwargs.get("exclude")) or [] if kwargs.get("dbt"): + self.dbt = DbtInvocation( + dbt_project_dir=kwargs.get("dbt_project_dir"), + dbt_target=kwargs.get("dbt_target"), + ) select = self.__get_selection(**kwargs) exclude = [] + + if kwargs.get("dbt_auto_artifacts"): + self.dbt.get_artifacts_for_erd() + artifacts_dir = f"{dbt_project_dir}/target" else: unsupported, rule = has_unsupported_rule( rules=select.extend(exclude) if exclude else select @@ -55,6 +66,7 @@ def evaluate_kwargs(self, **kwargs) -> dict: logger.error(message) raise click.UsageError(message) + logger.info(f"Using dbt artifact dir at: {artifacts_dir}") kwargs["artifacts_dir"] = artifacts_dir kwargs["dbt_project_dir"] = dbt_project_dir kwargs["select"] = select @@ -91,10 +103,10 @@ def __get_dir(self, **kwargs) -> str: def __get_selection(self, **kwargs): """Override the Selection using dbt's one with `--dbt`""" - return DbtInvocation( - dbt_project_dir=kwargs.get("dbt_project_dir"), - dbt_target=kwargs.get("dbt_target"), - ).get_selection( + if not self.dbt: + raise click.UsageError("Flag `--dbt` need to be enabled") + + return self.dbt.get_selection( select_rules=kwargs.get("select"), exclude_rules=kwargs.get("exclude"), ) diff --git a/dbterd/adapters/dbt_invocation.py b/dbterd/adapters/dbt_invocation.py index fb90dd1..ccf73b1 100644 --- a/dbterd/adapters/dbt_invocation.py +++ b/dbterd/adapters/dbt_invocation.py @@ -26,8 +26,52 @@ def __init__(self, dbt_project_dir: str = None, dbt_target: str = None) -> None: dbt_project_dir or os.environ.get("DBT_PROJECT_DIR") or str(Path.cwd()) ) self.target = dbt_target + self.args = ["--quiet", "--log-level", "none"] + + def __invoke(self, runner_args: List[str] = []): + """Base function of the dbt invocation + + Args: + runner_args (List[str], optional): Actual dbt arguments. Defaults to []. + + Raises: + click.UsageError: Invocation failed for a reason + + Returns: + dbtRunnerResult.result: dbtRunnerResult.result + """ + args = self.__construct_arguments(*runner_args) + logger.debug(f"Invoking: `dbt {' '.join(args)}` at {self.project_dir}") + r: dbtRunnerResult = self.dbt.invoke(args) + + if not r.success: + logger.error(str(r)) + raise click.UsageError(str(r)) + + return r.result + + def __construct_arguments(self, *args) -> List[str]: + """Enrich the dbt arguements with the based options + + Returns: + List[str]: Actual dbt arguments + """ + evaluated_args = args + if self.args: + evaluated_args = [*self.args, *args] + if self.project_dir: + evaluated_args.extend(["--project-dir", self.project_dir]) + if self.target: + evaluated_args.extend(["--target", self.target]) + + return evaluated_args def __ensure_dbt_installed(self): + """Verify if dbt get installed + + Raises: + click.UsageError: dbt is not installed + """ dbt_spec = importlib.util.find_spec("dbt") if dbt_spec and dbt_spec.loader: installed_path = dbt_spec.submodule_search_locations[0] @@ -55,22 +99,22 @@ def get_selection( Returns: List[str]: Selected node names with 'exact' rule """ - args = ["ls", "--project-dir", self.project_dir, "--resource-type", "model"] + args = ["ls", "--resource-type", "model"] if select_rules: args.extend(["--select", " ".join(select_rules)]) if exclude_rules: args.extend(["--exclude", " ".join(exclude_rules)]) - if self.target: - args.extend(["--target", self.target]) - - logger.info(f"Invoking: `dbt {' '.join(args)}` at {self.project_dir}") - r: dbtRunnerResult = self.dbt.invoke(args) - - if not r.success: - logger.error(str(r)) - raise click.UsageError("str(r)") + result = self.__invoke(runner_args=args) return [ f"exact:model.{str(x).split('.')[0]}.{str(x).split('.')[-1]}" - for x in r.result + for x in result ] + + def get_artifacts_for_erd(self): + """Generate dbt artifacts using `dbt docs generate` command + + Returns: + dbtRunnerResult: dbtRunnerResult + """ + return self.__invoke(runner_args=["docs", "generate"]) diff --git a/dbterd/cli/params.py b/dbterd/cli/params.py index 45f8cfc..c82a2d0 100644 --- a/dbterd/cli/params.py +++ b/dbterd/cli/params.py @@ -97,6 +97,13 @@ def common_params(func): default=None, type=click.STRING, ) + @click.option( + "--dbt-auto-artifacts", + help="Flag to force generating dbt artifact files leveraging Programmatic Invocation", + is_flag=True, + default=False, + show_default=True, + ) @functools.wraps(func) def wrapper(*args, **kwargs): return func(*args, **kwargs) # pragma: no cover diff --git a/docs/nav/guide/cli-references.md b/docs/nav/guide/cli-references.md index b4f1746..40b62a5 100644 --- a/docs/nav/guide/cli-references.md +++ b/docs/nav/guide/cli-references.md @@ -18,7 +18,7 @@ run Run the convert
-## run +## dbterd run Command to generate diagram-as-a-code file @@ -58,44 +58,7 @@ Command to generate diagram-as-a-code file -h, --help Show this message and exit. ``` -### --artifacts-dir (-ad) - -Configure the path to directory containing dbt artifact files. - -It will take the the nested `/target` directory of `--dbt-project-dir` if not specified. - -> Default to the current directory's `/target` if both this option and `--dbt-project-dir` option are not specified - -**Examples:** -=== "CLI" - - ```bash - dbterd run -ad "./target" - ``` -=== "CLI (long style)" - - ```bash - dbterd run --artifacts-dir "./target" - ``` - -### --output (-o) - -Configure the path to directory containing the output diagram file. -> Default to `./target` - -**Examples:** -=== "CLI" - - ```bash - dbterd run -o "./target" - ``` -=== "CLI (long style)" - - ```bash - dbterd run --output "./target" - ``` - -### --select (-s) +### dbterd run --select (-s) Selection criteria. > Select all dbt models if not specified, supports mulitple options @@ -103,6 +66,7 @@ Selection criteria. Rules: - By `name`: model name starts with input string +- By `exact`: exact model name, formed as `exact:model.package.name` - By `schema`: schema name starts with an input string, formed as `schema:` - By `wildcard`: model name matches to a [wildcard pattern](https://docs.python.org/3/library/fnmatch.html), formed as `wildcard:` - By `exposure`: exposure name, exact match @@ -111,7 +75,12 @@ Rules: === "CLI (by name)" ```bash - dbterd run -s "model.package_name.model_partital_name" + dbterd run -s "model.package.partital_name" + ``` +=== "CLI (by exact name)" + + ```bash + dbterd run -s "exact:model.package.name" ``` === "CLI (by schema)" @@ -121,12 +90,12 @@ Rules: === "CLI (by wildcard)" ```bash - dbterd run --select "wildcard:*xyz" + dbterd run -s "wildcard:*xyz" ``` === "CLI (by exposure)" ```bash - dbterd run --select "exposure:my_exposure_name" + dbterd run -s "exposure:my_exposure_name" ``` #### `AND` and `OR` logic @@ -148,7 +117,7 @@ Rules: dbterd run -s schema:abc -s wildcard:*xyz.* ``` -### --exclude (-ns) +### dbterd run --exclude (-ns) Exclusion criteria. Rules are the same as Selection Criteria. > Do not exclude any dbt models if not specified, supports mulitple options @@ -165,7 +134,44 @@ Exclusion criteria. Rules are the same as Selection Criteria. dbterd run --exclude 'model.package_name.table' ``` -### --target (-t) +### dbterd run --artifacts-dir (-ad) + +Configure the path to directory containing dbt artifact files. + +It will take the the nested `/target` directory of `--dbt-project-dir` if not specified. + +> Default to the current directory's `/target` if both this option and `--dbt-project-dir` option are not specified + +**Examples:** +=== "CLI" + + ```bash + dbterd run -ad "./target" + ``` +=== "CLI (long style)" + + ```bash + dbterd run --artifacts-dir "./target" + ``` + +### dbterd run --output (-o) + +Configure the path to directory containing the output diagram file. +> Default to `./target` + +**Examples:** +=== "CLI" + + ```bash + dbterd run -o "./target" + ``` +=== "CLI (long style)" + + ```bash + dbterd run --output "./target" + ``` + +### dbterd run --target (-t) Target to the diagram-as-code platform > Default to `dbml` @@ -184,7 +190,7 @@ Supported target, please visit [Generate the Targets](https://dbterd.datnguyen.d dbterd run --target dbml ``` -### --algo (-a) +### dbterd run --algo (-a) Specified algorithm in the way to detect diagram connectors > Default to `test_relationship` @@ -226,7 +232,7 @@ In the above: dbterd run --algo "test_relationship:(name:foreign_key|c_from:fk_column_name|c_to:pk_column_name)" ``` -### --manifest-version (-mv) +### dbterd run --manifest-version (-mv) Specified dbt manifest.json version > Auto detect if not specified @@ -243,7 +249,7 @@ Specified dbt manifest.json version dbterd run -mv 7 ``` -### --catalog-version (-cv) +### dbterd run --catalog-version (-cv) Specified dbt catalog.json version > Auto detect if not specified @@ -260,7 +266,7 @@ Specified dbt catalog.json version dbterd run -cv 7 ``` -### --resource-type (-rt) +### dbterd run --resource-type (-rt) Specified dbt resource type(seed, model, source, snapshot). > Default to `["model"]`, supports mulitple options @@ -277,7 +283,7 @@ Specified dbt resource type(seed, model, source, snapshot). dbterd run --resource-type model ``` -### --dbt +### dbterd run --dbt Flag to indicate the Selecton to follow dbt's one leveraging Programmatic Invocation > Default to `False` @@ -296,7 +302,22 @@ Flag to indicate the Selecton to follow dbt's one leveraging Programmatic Invoca # select starts with 'something' ``` -### --dbt-project-dir (-dpd) +### dbterd run --dbt --dbt-auto-artifact + +Flag to indicate force running `dbt docs generate` to the targetted project in order to produce the dbt artifact files. + +This option have to be enabled together with `--dbt` option, and will override the value of `--artifacts-dir` to be using the `/target` dir of the value of `--dbt-project-dir`. + +> Default to `False` + +**Examples:** +=== "CLI" + + ```bash + dbterd run -s +something --dbt --dbt-auto-artifacts + ``` + +### dbterd run --dbt-project-dir (-dpd) Specified dbt project directory path @@ -313,7 +334,7 @@ You should specified this option if your CWD is not the dbt project dir, and nor # the artifacts dir will probably be assumed as: /path/to/dbt/project/target ``` -### --dbt-target (-dt) +### dbterd run --dbt-target (-dt) Specified dbt target name @@ -330,9 +351,9 @@ Probably used with `--dbt` enabled. # the artifacts dir will probably be assumed as: /path/to/dbt/project/target ``` -## debug +## dbterd debug -Shows hidden configured values +Shows hidden configured values, which will help us to see what configs are passed into and how they are evaluated to be used. **Examples:** === "Output" diff --git a/tests/unit/adapters/test_base.py b/tests/unit/adapters/test_base.py index 6b96051..178095b 100644 --- a/tests/unit/adapters/test_base.py +++ b/tests/unit/adapters/test_base.py @@ -6,6 +6,7 @@ from dbterd import default from dbterd.adapters.base import Executor +from dbterd.adapters.dbt_invocation import DbtInvocation class TestBase: @@ -50,11 +51,20 @@ def test___read_catalog(self): ) def test__get_selection(self, mock_dbt_invocation): worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy"))) + worker.dbt = DbtInvocation() assert "dummy" == worker._Executor__get_selection( select_rules=[], exclude_rules=[] ) mock_dbt_invocation.assert_called_once() + @mock.patch( + "dbterd.adapters.base.DbtInvocation.get_selection", return_value="dummy" + ) + def test__get_selection__error(self, mock_dbt_invocation): + worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy"))) + with pytest.raises(click.UsageError): + worker._Executor__get_selection() + @pytest.mark.parametrize( "kwargs, expected", [ @@ -80,16 +90,37 @@ def test__get_selection(self, mock_dbt_invocation): exclude=[], ), ), + ( + dict(select=[], exclude=[], dbt=True, dbt_auto_artifacts=True), + dict( + dbt=True, + dbt_auto_artifacts=True, + artifacts_dir="/path/dpd/target", + dbt_project_dir="/path/dpd", + select=["yolo"], + exclude=[], + ), + ), ], ) @mock.patch("dbterd.adapters.base.Executor._Executor__get_dir") @mock.patch("dbterd.adapters.base.Executor._Executor__get_selection") - def test_evaluate_kwargs(self, mock_get_selection, mock_get_dir, kwargs, expected): + @mock.patch("dbterd.adapters.base.DbtInvocation.get_artifacts_for_erd") + def test_evaluate_kwargs( + self, + mock_get_artifacts_for_erd, + mock_get_selection, + mock_get_dir, + kwargs, + expected, + ): worker = Executor(ctx=click.Context(command=click.BaseCommand("dummy"))) mock_get_dir.return_value = ("/path/ad", "/path/dpd") mock_get_selection.return_value = ["yolo"] assert expected == worker.evaluate_kwargs(**kwargs) mock_get_dir.assert_called_once() + if kwargs.get("dbt_auto_artifacts"): + mock_get_artifacts_for_erd.assert_called_once() @pytest.mark.parametrize( "kwargs, mock_isfile_se, expected", diff --git a/tests/unit/adapters/test_dbt_invocation.py b/tests/unit/adapters/test_dbt_invocation.py index 5fdf354..0b3f814 100644 --- a/tests/unit/adapters/test_dbt_invocation.py +++ b/tests/unit/adapters/test_dbt_invocation.py @@ -49,12 +49,18 @@ def test_get_selection( ) assert actual == expected - args = ["ls", "--project-dir", invoker.project_dir, "--resource-type", "model"] + args = ["ls", "--resource-type", "model"] if select_rules: args.extend(["--select", " ".join(select_rules)]) if exclude_rules: args.extend(["--exclude", " ".join(exclude_rules)]) - mock_dbtRunner_invoke.assert_called_once_with(args) + mock_dbtRunner_invoke.assert_called_once_with( + [ + *["--quiet", "--log-level", "none"], + *args, + *["--project-dir", invoker.project_dir], + ] + ) @mock.patch("dbterd.adapters.dbt_invocation.dbtRunner.invoke") def test_get_selection__failed(self, mock_dbtRunner_invoke): @@ -63,3 +69,17 @@ def test_get_selection__failed(self, mock_dbtRunner_invoke): DbtInvocation(dbt_target="dummy").get_selection( select_rules=[], exclude_rules=[] ) + + @mock.patch("dbterd.adapters.dbt_invocation.dbtRunner.invoke") + def test_get_artifacts_for_erd(self, mock_dbtRunner_invoke): + invoker = DbtInvocation() + _ = invoker.get_artifacts_for_erd() + + args = ["docs", "generate"] + mock_dbtRunner_invoke.assert_called_once_with( + [ + *["--quiet", "--log-level", "none"], + *args, + *["--project-dir", invoker.project_dir], + ] + )