Refactor dbt project structure and add silver fact review model (#17)

andrewcrosher · Jan 26, 2025 · 5e14535 · 5e14535
1 parent 1c6a21e
commit 5e14535
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 67 deletions.
diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,12 +1,16 @@
-name: 'ajc'
+name: 'ajc_dbt'
+
 version: '0.0.1'
+
 profile: 'ajc_adb_dbt_all_purpose'
 model-paths: ["models"]
 seed-paths: ["seeds"]
+
 clean-targets:
   - "target"
   - "dbt_packages"
+
 models:
   +persist_docs:
     relation: true
-    columns: true
+    columns: true
diff --git a/models/bronze_albums.yml b/models/bronze_albums.yml
@@ -0,0 +1,26 @@
+version: 2
+
+sources:
+  - name: albums
+    schema: albums
+    tables:
+      - name: ajc_albums
+
+models:
+  - name: bronze_albums
+    description: a bronze dbt model of the raw albums info
+    columns:
+      - name: currentAlbum
+        data_type: struct<amazonmusicid:string,applemusicid:string,artist:string,artistorigin:string,deezerid:string,genres:array<string>,globalreviewsurl:string,images:array<struct<height:bigint,url:string,width:bigint>>,name:string,qobuzid:string,releasedate:string,slug:string,spotifyid:string,subgenres:array<string>,tidalid:bigint,wikipediaurl:string,youtubemusicid:string>
+      - name: currentAlbumNotes
+        data_type: string
+      - name: history
+        data_type: array<struct<album:struct<amazonmusicid:string,applemusicid:string,artist:string,artistorigin:string,deezerid:string,genres:array<string>,globalreviewsurl:string,images:array<struct<height:bigint,url:string,width:bigint>>,name:string,qobuzid:string,releasedate:string,slug:string,spotifyid:string,subgenres:array<string>,tidalid:bigint,wikipediaurl:string,youtubemusicid:string>,generatedat:string,globalrating:double,rating:bigint,revealedalbum:boolean,review:string>>
+      - name: name
+        data_type: string
+      - name: shareableUrl
+        data_type: string
+      - name: updateFrequency
+        data_type: string
+      - name: load_timestamp
+        data_type: timestamp
diff --git a/models/gold_fact_review.yml b/models/gold_fact_review.yml
@@ -34,3 +34,16 @@ models:
         description: The date of the review
         data_type: date
         quote: true
+
+exposures:
+  - name: 1001_albums_review_dashboard
+    type: dashboard
+    maturity: high
+    url: https://adb-2359489148887710.10.azuredatabricks.net/dashboardsv3/01ef83092f1b1403b7967bea7000d543/published?o=2359489148887710&f_01ef8314636c17ee88df636697944312=_all_&f_01ef8315346e1f7e9848122c041edf4b=_all_
+    description: >
+      A Databricks dashboard
+    depends_on:
+      - ref('gold_fact_review')
+    owner:
+      name: Andrew Crosher
+      email: andrewcrosher@outlook.com
diff --git a/models/silver_dim_abum.yml → models/silver_dim_album.yml b/models/silver_dim_abum.yml → models/silver_dim_album.yml
diff --git a/models/albums_properties.yml → models/silver_dim_date.yml b/models/albums_properties.yml → models/silver_dim_date.yml
@@ -1,57 +1,5 @@
 version: 2
-sources:
-  - name: albums
-    schema: albums
-    tables:
-      - name: ajc_albums
 models:
-  - name: bronze_albums
-    description: a bronze dbt model of the raw albums info
-    columns:
-      - name: currentAlbum
-        data_type: struct<amazonmusicid:string,applemusicid:string,artist:string,artistorigin:string,deezerid:string,genres:array<string>,globalreviewsurl:string,images:array<struct<height:bigint,url:string,width:bigint>>,name:string,qobuzid:string,releasedate:string,slug:string,spotifyid:string,subgenres:array<string>,tidalid:bigint,wikipediaurl:string,youtubemusicid:string>
-      - name: currentAlbumNotes
-        data_type: string
-      - name: history
-        data_type: array<struct<album:struct<amazonmusicid:string,applemusicid:string,artist:string,artistorigin:string,deezerid:string,genres:array<string>,globalreviewsurl:string,images:array<struct<height:bigint,url:string,width:bigint>>,name:string,qobuzid:string,releasedate:string,slug:string,spotifyid:string,subgenres:array<string>,tidalid:bigint,wikipediaurl:string,youtubemusicid:string>,generatedat:string,globalrating:double,rating:bigint,revealedalbum:boolean,review:string>>
-      - name: name
-        data_type: string
-      - name: shareableUrl
-        data_type: string
-      - name: updateFrequency
-        data_type: string
-      - name: load_timestamp
-        data_type: timestamp
-
-  - name: silver_fact_review
-    description: a fact table of album review measures
-    columns:
-      - name: youtube_id
-        description: The PK of the table, the YouTube album ID
-        data_type: string
-        tests:
-          - not_null
-          - unique
-      - name: my_rating
-        description: My rating
-        data_type: bigint
-        tests:
-          - accepted_values:
-              values:
-                - "1"
-                - "2"
-                - "3"
-                - "4"
-                - "5"
-      - name: global_rating
-        description: The global rating
-        data_type: double
-      - name: review_date
-        description: The date when the review was logged
-        data_type: date
-        tests:
-          - not_null
-
   - name: silver_dim_date
     description: a date dimension, materialised as a table using the dbt_date.get_date_dimension macro
     columns:
@@ -163,16 +111,3 @@ models:
       - name: year_end_date
         data_type: date
         quote: true
-
-exposures:
-  - name: 1001_albums_review_dashboard
-    type: dashboard
-    maturity: high
-    url: https://adb-2359489148887710.10.azuredatabricks.net/dashboardsv3/01ef83092f1b1403b7967bea7000d543/published?o=2359489148887710&f_01ef8314636c17ee88df636697944312=_all_&f_01ef8315346e1f7e9848122c041edf4b=_all_
-    description: >
-      A Databricks dashboard
-    depends_on:
-      - ref('gold_fact_review')
-    owner:
-      name: Andrew Crosher
-      email: andrewcrosher@outlook.com
diff --git a/models/silver_fact_review.yml b/models/silver_fact_review.yml
@@ -0,0 +1,30 @@
+version: 2
+models:
+  - name: silver_fact_review
+    description: a fact table of album review measures
+    columns:
+      - name: youtube_id
+        description: The PK of the table, the YouTube album ID
+        data_type: string
+        tests:
+          - not_null
+          - unique
+      - name: my_rating
+        description: My rating
+        data_type: bigint
+        tests:
+          - accepted_values:
+              values:
+                - "1"
+                - "2"
+                - "3"
+                - "4"
+                - "5"
+      - name: global_rating
+        description: The global rating
+        data_type: double
+      - name: review_date
+        description: The date when the review was logged
+        data_type: date
+        tests:
+          - not_null