From 5ec96001ae9757c9b57939b833e644dea0025d65 Mon Sep 17 00:00:00 2001
From: "Wei-Chun, Chang" <wcchang@infuseai.io>
Date: Mon, 20 May 2024 12:03:28 +0800
Subject: [PATCH 1/2] Add recce staging env preparation workflow

Signed-off-by: Wei-Chun, Chang <wcchang@infuseai.io>
---
 .github/workflows/recce-staging-daily.yml | 71 +++++++++++++++++++++++
 1 file changed, 71 insertions(+)
 create mode 100644 .github/workflows/recce-staging-daily.yml

diff --git a/.github/workflows/recce-staging-daily.yml b/.github/workflows/recce-staging-daily.yml
new file mode 100644
index 000000000..b246cef91
--- /dev/null
+++ b/.github/workflows/recce-staging-daily.yml
@@ -0,0 +1,71 @@
+name: OSO Recce Staging CI
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 18 * * *' # run at 2 AM (UTC + 8) everyday
+  pull_request:
+    types: [closed]
+    branches: [main, dev]
+    paths:
+      - warehouse/dbt/**
+
+env:
+  # dbt env variables used in your dbt profiles.yml
+  DBT_PROFILES_DIR: ./
+  DBT_GOOGLE_PROJECT: ${{ vars.DBT_GOOGLE_PROJECT }}
+  DBT_GOOGLE_DATASET: ${{ vars.DBT_GOOGLE_DATASET }}
+  DBT_GOOGLE_DEV_DATASET: ${{ vars.DBT_GOOGLE_DEV_DATASET }}
+  DBT_GOOGLE_KEYFILE: /tmp/google/google-service-account.json
+  PACKAGES_YAML: ${{ vars.PACKAGES_YAML }}
+  KEYFILE_CONTENTS: ${{ secrets.KEYFILE_CONTENTS }}
+
+jobs:
+  prepare-recce-staging-env:
+    name: Prepare Recce staging env
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12.x"
+      
+      - name: Install dependencies
+        run: |
+          pipx install poetry==1.7.1
+          poetry install
+          source $(poetry env info --path)/bin/activate
+          which dbt
+      
+      - name: Add packages.yml file
+        run: |
+          echo '${{ vars.PACKAGES_YAML }}' > packages.yml
+      
+      - name: Prep Google keyfile
+        run: | 
+          mkdir -p "$(dirname $DBT_GOOGLE_KEYFILE)" 
+          echo "$KEYFILE_CONTENTS" > $DBT_GOOGLE_KEYFILE
+      
+      - name: Prepare dbt Base environment
+        run: |
+          source $(poetry env info --path)/bin/activate
+          git checkout ${{ github.event.pull_request.base.sha }}
+          dbt deps
+          dbt seed --target ${{ env.DBT_BASE_TARGET }}
+          dbt run --target ${{ env.DBT_BASE_TARGET }}
+          dbt docs generate --target ${{ env.DBT_BASE_TARGET }}
+        env:
+          DBT_BASE_TARGET: "playground"
+      
+      - name: Archive DBT artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: dbt-artifacts
+          path: |
+            target/manifest.json
+            target/catalog.json

From 8f3bf78fac6472d91b5d2bcc350fffa9b39a2503 Mon Sep 17 00:00:00 2001
From: "Wei-Chun, Chang" <wcchang@infuseai.io>
Date: Mon, 20 May 2024 14:52:40 +0800
Subject: [PATCH 2/2] Refine workflow

Signed-off-by: Wei-Chun, Chang <wcchang@infuseai.io>
---
 .github/workflows/recce-staging-daily.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/recce-staging-daily.yml b/.github/workflows/recce-staging-daily.yml
index b246cef91..11a61026e 100644
--- a/.github/workflows/recce-staging-daily.yml
+++ b/.github/workflows/recce-staging-daily.yml
@@ -15,7 +15,6 @@ env:
   DBT_PROFILES_DIR: ./
   DBT_GOOGLE_PROJECT: ${{ vars.DBT_GOOGLE_PROJECT }}
   DBT_GOOGLE_DATASET: ${{ vars.DBT_GOOGLE_DATASET }}
-  DBT_GOOGLE_DEV_DATASET: ${{ vars.DBT_GOOGLE_DEV_DATASET }}
   DBT_GOOGLE_KEYFILE: /tmp/google/google-service-account.json
   PACKAGES_YAML: ${{ vars.PACKAGES_YAML }}
   KEYFILE_CONTENTS: ${{ secrets.KEYFILE_CONTENTS }}
@@ -27,8 +26,6 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
       
       - name: Set up Python
         uses: actions/setup-python@v4
@@ -54,10 +51,8 @@ jobs:
       - name: Prepare dbt Base environment
         run: |
           source $(poetry env info --path)/bin/activate
-          git checkout ${{ github.event.pull_request.base.sha }}
           dbt deps
-          dbt seed --target ${{ env.DBT_BASE_TARGET }}
-          dbt run --target ${{ env.DBT_BASE_TARGET }}
+          dbt build --target ${{ env.DBT_BASE_TARGET }}
           dbt docs generate --target ${{ env.DBT_BASE_TARGET }}
         env:
           DBT_BASE_TARGET: "playground"