Update dagster (#11)

* cleanup

* update to new dagster version, use DbtProject

* use custom dbt_resource

* docstrings

.env

@@ -1,5 +1,2 @@
 # Location to store both dagster and data in (can be relative)
 DATA_DIR=data
-
-# parse dbt project during runtime (useful for dev, but slows down asset execution)
-DAGSTER_DBT_PARSE_PROJECT_ON_LOAD=1

Makefile

@@ -21,7 +21,6 @@ requirements: .venv  ## Install/refresh Python project requirements
 
 .PHONY: build
 build:
-	$(VENV_BIN)/uv pip install build
 	$(VENV_BIN)/python -m build
 
 .PHONY: dagster

README.md

@@ -29,17 +29,10 @@ Commands:
     make requirements
     source .venv/bin/activate
     ```
-* PowerShell:
+* Windows:
     ```powershell
     python -m venv .venv
-    .venv\Scripts\Activate.ps1
-    python -m pip install --upgrade uv
-    uv pip install --editable .[dev]
-    ```
-* Windows CMD:
-    ```
-    python -m venv .venv
-    .venv\Scripts\activate.bat
+    .venv\Scripts\activate
     python -m pip install --upgrade uv
     uv pip install --editable .[dev]
     ```

foneplatform/assets/dbt.py

@@ -6,15 +6,15 @@
     dbt_assets,
 )
 
-from ..resources.dbt_resource import DBT_MANIFEST_PATH
+from ..resources.dbt_resource import dbt_project
 
 
 class DbtConfig(Config):  # type: ignore
     full_refresh: bool = False
 
 
 @dbt_assets(
-    manifest=DBT_MANIFEST_PATH,
+    manifest=dbt_project.manifest_path,
     dagster_dbt_translator=DagsterDbtTranslator(
         settings=DagsterDbtTranslatorSettings(enable_asset_checks=True)
     ),

foneplatform/resources/dbt_resource.py

@@ -2,16 +2,16 @@
 import sys
 from pathlib import Path
 
-from dagster_dbt import DbtCliResource
+from dagster_dbt import DbtCliResource, DbtProject
 
-dbt_project_dir = Path(__file__).parent.parent.parent.joinpath("dbt").resolve()
-DBT_MANIFEST_PATH = dbt_project_dir.joinpath("target", "manifest.json")
+dbt_project = DbtProject(project_dir=Path(__file__).parent.parent.parent.joinpath("dbt").resolve())
 
 
 def get_dbt_executable() -> str:
     """
-    If we're in a virtualenv, we want to use the dbt executable in the virtualenv.
     Returns the path to the dbt executable.
+
+    If we're in a virtualenv, we want to use the dbt executable in the virtualenv.
     """
     if sys.prefix != sys.base_prefix:
         if os.name == "nt":
@@ -26,20 +26,12 @@ def get_dbt_resource() -> DbtCliResource:
     """
     Returns a DbtCliResource.
 
-    If DAGSTER_DBT_PARSE_PROJECT_ON_LOAD is set, a manifest will be created at run time.
+    On 'dagster dev', a manifest will be created at run time.
     """
     dbt_resource = DbtCliResource(
-        project_dir=os.fspath(dbt_project_dir),
-        profiles_dir=os.fspath(dbt_project_dir),
+        project_dir=dbt_project,
+        profiles_dir=os.fspath(dbt_project.project_dir),
         dbt_executable=get_dbt_executable(),
     )
-    if os.getenv("DAGSTER_DBT_PARSE_PROJECT_ON_LOAD"):
-        (
-            dbt_resource.cli(
-                ["--quiet", "parse"],
-                target_path=Path("target"),
-            )
-            .wait()
-            .target_path.joinpath("manifest.json")
-        )
+    dbt_project.prepare_if_dev()
     return dbt_resource

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "f1dbtduckdb"
-version = "0.5.0"
+version = "0.6.0"
 description = "Experimenting with dbt/DuckDB using Ergast API as main source"
 readme = "README.md"
 requires-python = ">= 3.10"
@@ -18,26 +18,24 @@ classifiers = [
 ]
 
 dependencies = [
-  "dagster-dbt>=0.23.0, <0.24.0",
-  "dagster-webserver>=1.7.0, <1.8.0",
-  "dbt-duckdb>=1.8.0, <1.9.0",
+  "dagster-dbt>=0.23.0",
+  "dagster-webserver>=1.7.0, <1.10.0",
+  "dbt-duckdb>=1.8.0, <1.10.0",
   "duckdb>=1.0.0, <2.0.0",
 ]
 
 [project.optional-dependencies]
 dev = [
   # development & testing tools
   "mypy>=1.0.0, <2.0.0",
-  "ruff>=0.5.0, <0.6.0",
+  "ruff>=0.5.0",
   "sqlfluff-templater-dbt>=3.0.0, <4.0.0",
+  "build>=1.0.0, <2.0.0",
 ]
 
-[tool.dagster]
-module_name = "foneplatform"
-code_location_name = "foneplatform"
-
-[tool.setuptools]
-packages = ["foneplatform", "dbt"]
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["foneplatform*", "dbt*"]
 
 [tool.setuptools.package-data]
 dbt = [
@@ -48,6 +46,10 @@ dbt = [
   "models/stage/ergast/*",
 ]
 
+[tool.dagster]
+module_name = "foneplatform"
+code_location_name = "foneplatform"
+
 [[tool.mypy.overrides]]
 module = "ergast,staging,requests.adapters"
 ignore_missing_imports = true