Merge pull request #1 from ndrewwm/pace-of-discovery

Streamline project DAG, add report tracking counts of "new" artists/tracks

Author: ndrewwm

dbt_spotify/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'dbt_spotify'
-version: '1.0.0'
+version: '2.0.0'
 
 profile: 'dbt_spotify'
 
@@ -18,7 +18,12 @@ models:
   dbt_spotify:
     staging:
       +materialized: view
-    intermediate:
-      +materialized: ephemeral
+      +docs:
+        node_color: 'silver'
     marts:
       +materialized: table
+      +docs:
+        node_color: salmon
+      reports:
+        +docs:
+          node_color: skyblue

dbt_spotify/models/intermediate/int_dim_track.sql

@@ -0,0 +1,15 @@
+{% docs dttm_first_played %}
+Date and time (America/Boise) when the track or artist was first recorded as being played.
+{% enddocs %}
+
+{% docs yr_first_played %}
+Year when the track or artist was first recorded as being played.
+{% enddocs %}
+
+{% docs wk_first_played %}
+Week (integer) when the track or artist was first recorded as being played.
+{% enddocs %}
+
+{% docs total_plays %}
+Total number of times the track or artist has been recorded as being played.
+{% enddocs %}

dbt_spotify/models/intermediate/int_fct_played_track.sql

@@ -11,22 +11,34 @@ models:
           - not_null
 
       - name: album
-        description: Album title.
+        description: '{{ doc("track_album") }}'
 
       - name: release_date
-        description: Date when the album was released.
+        description: '{{ doc("album_release_date") }}'
 
   - name: dim_artist
-    description: Directory of artists observed in the data. Each row is an artist or performing group.
+    description: Directory of artists observed in the data. Each row is an artist or performing group (or set of artists).
     columns:
       - name: artist_id
         description: Unique artist ID.
         tests:
           - unique
           - not_null
 
-      - name: artist
-        description: Artist/performer name.
+      - name: artists
+        description: '{{ doc("track_artists") }}'
+
+      - name: total_plays
+        description: '{{ doc("total_plays") }}'
+
+      - name: dttm_first_played
+        description: '{{ doc("dttm_first_played") }}'
+
+      - name: yr_first_played
+        description: '{{ doc("yr_first_played") }}'
+
+      - name: wk_first_played
+        description: '{{ doc("wk_first_played") }}'
 
   - name: dim_track
     description: Directory of all tracks observed. Each row is a track.
@@ -43,11 +55,26 @@ models:
       - name: album_id
         description: Album ID corresponding to this track.
 
+      - name: album
+        description: '{{ doc("track_album") }}'
+
       - name: artists
         description: '{{ doc("track_artists") }}'
 
       - name: duration_ms
-        description: Track duration (milliseconds).
+        description: '{{ doc("duration_ms") }}'
+
+      - name: total_plays
+        description: '{{ doc("total_plays") }}'
+
+      - name: dttm_first_played
+        description: '{{ doc("dttm_first_played") }}'
+
+      - name: yr_first_played
+        description: '{{ doc("yr_first_played") }}'
+
+      - name: wk_first_played
+        description: '{{ doc("wk_first_played") }}'
 
   - name: fct_played_track
     description: Spotify listening history. Each row is an instance that a track was played.
@@ -67,8 +94,11 @@ models:
       - name: played_at
         description: '{{ doc("played_at") }}'
 
+      - name: played_at_mtn
+        description: '{{ doc("played_at_mtn") }}'
+
       - name: track_popularity
         description: '{{ doc("popularity") }}'
 
       - name: context
-        description: What context the track is being played in.
+        description: '{{ doc("context") }}'

dbt_spotify/models/marts/_spotify__docs.md

@@ -1 +1,10 @@
-select * from {{ ref("stg_dim_album") }}
+with tracks as (
+    select * from {{ ref("stg_recent_tracks") }}
+)
+
+select distinct
+    album_id,
+    track_album as album,
+    album_release_date as release_date
+from tracks
+order by album_id

dbt_spotify/models/marts/_spotify__models.yml

@@ -1 +1,24 @@
-select * from {{ ref("stg_dim_artist") }}
+with tracks as (
+    select * from {{ ref("stg_recent_tracks") }}
+),
+
+summary as (
+    select
+        artist_id,
+        count(*) as total_plays,
+        min(played_at_mtn) as dttm_first_played
+    from tracks
+    group by artist_id
+)
+
+select distinct
+    tr.artist_id,
+    tr.track_artists as artists,
+    su.total_plays,
+    su.dttm_first_played,
+    year(su.dttm_first_played::date) as yr_first_played,
+    week(su.dttm_first_played::date) as wk_first_played
+from tracks tr
+left join summary su
+    on tr.artist_id = su.artist_id
+order by tr.artist_id

dbt_spotify/models/marts/dim_album.sql

@@ -1 +1,28 @@
-select * from {{ ref("int_dim_track") }}
+with tracks as (
+    select * from {{ ref("stg_recent_tracks") }}
+),
+
+summary as (
+    select
+        track_id,
+        count(*) as total_plays,
+        min(played_at_mtn) as dttm_first_played
+    from tracks
+    group by track_id
+)
+
+select distinct
+    tr.track_id,
+    tr.album_id,
+    tr.track_album as album,
+    tr.track_name,
+    tr.track_artists as artists,
+    su.total_plays,
+    su.dttm_first_played,
+    year(su.dttm_first_played::date) as yr_first_played,
+    week(su.dttm_first_played::date) as wk_first_played,
+    tr.duration_ms
+from tracks tr
+left join summary su
+    on tr.track_id = su.track_id
+order by tr.track_id

dbt_spotify/models/marts/dim_artist.sql

@@ -1 +1,18 @@
-select * from {{ ref('int_fct_played_track') }}
+with tracks_played as (
+    select * from {{ ref("stg_recent_tracks") }}
+),
+
+final as (
+    select
+        play_id,
+        track_id,
+        album_id,
+        played_at,
+        played_at_mtn,
+        track_popularity,
+        context
+    from tracks_played
+    order by played_at_mtn desc
+)
+
+select * from final

dbt_spotify/models/marts/dim_track.sql

@@ -3,6 +3,65 @@ version: 2
 models:
   - name: rpt_artist_counts
     description: A summary of the total number of plays and minutes listened over the prior 30 days, by artist.
+    columns:
+      - name: artists
+        description: '{{ doc("track_artists") }}'
+
+      - name: plays
+        description: Number of times a track from the artist(s) has been played.
+
+      - name: minutes_played
+        description: Total number of minutes spent listening to the artist(s) over the past 30 days.
 
   - name: rpt_track_counts
     description: A summary of the total number of plays and minutes listened over the prior 30 days, by track.
+    columns:
+      - name: track_name
+        description: The track's name.
+
+      - name: artists
+        description: '{{ doc("track_artists") }}'
+
+      - name: album
+        description: '{{ doc("track_album") }}'
+
+      - name: plays
+        description: Number of times this track has been played in the past 30 days.
+
+      - name: minutes_played
+        description: Total number of minutes spent listening to this track over the past 30 days.
+
+  - name: rpt_discovery_rate
+    description: |
+      A report, organized by week and year, tracking the total number of plays per week,
+      how many *unique* artists/tracks were played, and how many *new* artists/tracks were played.
+    columns:
+      - name: year_played
+        description: Year
+
+      - name: week_played
+        description: Week (integer)
+
+      - name: week_dt
+        description: Date corresponding to the week. Included for plotting convenience.
+
+      - name: plays
+        description: Total number of plays recorded during this week.
+
+      - name: uniq_artists
+        description: Count of unique artists observed.
+
+      - name: uniq_tracks
+        description: Count of unique tracks observed.
+
+      - name: new_tracks
+        description: Count of new tracks observed.
+
+      - name: new_artists
+        description: Count of new artists observed
+
+      - name: track_discovery_rate
+        description: "WIP. A proportion: new_tracks / uniq_tracks"
+
+      - name: artist_discovery_rate
+        description: "WIP. A proportion: new_artists / uniq_artists"

dbt_spotify/models/marts/fct_played_track.sql

@@ -0,0 +1,69 @@
+with tracks as (
+    select * from {{ ref("fct_played_track") }}
+),
+
+dim_track as (
+    select * from {{ ref("dim_track") }}
+),
+
+dim_artist as (
+    select * from {{ ref("dim_artist") }}
+),
+
+base as (
+    select distinct
+        year(played_at_mtn) as year_played,
+        week(played_at_mtn) as week_played,
+        max(played_at_mtn::date) as week_dt,
+        count(*) as plays,
+        count(distinct dt.artists) as uniq_artists,
+        count(distinct dt.track_id) as uniq_tracks
+    from tracks
+    left join dim_track dt
+        on tracks.track_id = dt.track_id
+    group by year_played, week_played
+    order by year_played, week_played
+),
+
+new_tracks as (
+    select
+        yr_first_played as year_played,
+        wk_first_played as week_played,
+        count(*) as new_tracks
+    from dim_track
+    group by yr_first_played, wk_first_played
+),
+
+new_artists as (
+    select
+        yr_first_played as year_played,
+        wk_first_played as week_played,
+        count(*) as new_artists
+    from dim_artist
+    group by yr_first_played, wk_first_played  
+),
+
+final as (
+    select
+        base.*,
+        new_tracks.new_tracks,
+        new_artists.new_artists,
+        {{ 
+            dbt_utils.safe_divide('new_tracks.new_tracks', 'base.uniq_tracks') 
+        }} as track_discovery_rate,
+        {{
+            dbt_utils.safe_divide('new_artists.new_artists', 'base.uniq_artists')
+        }} as artist_discovery_rate
+    from base
+    left join new_tracks
+        on 
+            base.year_played = new_tracks.year_played
+            and base.week_played = new_tracks.week_played
+
+    left join new_artists
+        on 
+            base.year_played = new_artists.year_played
+            and base.week_played = new_artists.week_played
+)
+
+select * from final