From 9cb55f0517986a82f3cf867cfb9367a803fbebb6 Mon Sep 17 00:00:00 2001
From: Sung Won Chung <sungwonchung3@gmail.com>
Date: Thu, 1 Aug 2024 09:41:11 -0700
Subject: [PATCH] Daily ELT job (#6)

* first draft

* adjust for cron

* run on main branch only

* add deps and append data command defaults to today's date
---
 .github/workflows/daily_elt_job.yml | 46 +++++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
 create mode 100644 .github/workflows/daily_elt_job.yml

diff --git a/.github/workflows/daily_elt_job.yml b/.github/workflows/daily_elt_job.yml
new file mode 100644
index 0000000..bf17f34
--- /dev/null
+++ b/.github/workflows/daily_elt_job.yml
@@ -0,0 +1,46 @@
+name: Daily ELT Job
+run-name: 🧃 Daily ELT Job 🧃
+
+on:
+  schedule:
+    - cron: '0 14 * * 1-5'  # 7am PST (14:00 UTC) Monday to Friday
+  workflow_dispatch:  # Allow manual triggering
+
+jobs:
+  daily_elt:
+    name: Daily ELT Process
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/main'  # Only run on the main branch
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Authenticate to Google Cloud
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: '${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}'
+
+      - name: Install SQLMesh Enterprise + Dependencies
+        run: |
+          pip install poetry keyring keyrings-google-artifactregistry-auth
+          pip install ruff==0.4.8 mypy==1.10.0 pytest==8.1.1 typer==0.12.3
+          pip install "sqlmesh-enterprise[bigquery,web,services,github]==0.27.0" --extra-index-url ${{ secrets.SQLMESH_ENTERPRISE_TOKEN }}
+        shell: bash
+
+      - name: Append raw events to tcloud_raw_data.raw_events to display observer metrics
+        run: |
+          python demo_scripts/main.py append-rawdata
+        env:
+          GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}
+
+      - name: Run SQLMesh
+        run: |
+          sqlmesh run
+        env:
+          TOBIKO_CLOUD_TOKEN: '${{ secrets.TOBIKO_CLOUD_TOKEN }}'
+          GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}
\ No newline at end of file