-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* first draft * adjust for cron * run on main branch only * add deps and append data command defaults to today's date
- Loading branch information
1 parent
668e9f3
commit 9cb55f0
Showing
1 changed file
with
46 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,46 @@ | ||
name: Daily ELT Job | ||
run-name: 🧃 Daily ELT Job 🧃 | ||
|
||
on: | ||
schedule: | ||
- cron: '0 14 * * 1-5' # 7am PST (14:00 UTC) Monday to Friday | ||
workflow_dispatch: # Allow manual triggering | ||
|
||
jobs: | ||
daily_elt: | ||
name: Daily ELT Process | ||
runs-on: ubuntu-latest | ||
if: github.ref == 'refs/heads/main' # Only run on the main branch | ||
steps: | ||
- name: Checkout repository | ||
uses: actions/checkout@v4 | ||
|
||
- name: Setup Python | ||
uses: actions/setup-python@v5 | ||
with: | ||
python-version: '3.12' | ||
|
||
- name: Authenticate to Google Cloud | ||
uses: 'google-github-actions/auth@v2' | ||
with: | ||
credentials_json: '${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}' | ||
|
||
- name: Install SQLMesh Enterprise + Dependencies | ||
run: | | ||
pip install poetry keyring keyrings-google-artifactregistry-auth | ||
pip install ruff==0.4.8 mypy==1.10.0 pytest==8.1.1 typer==0.12.3 | ||
pip install "sqlmesh-enterprise[bigquery,web,services,github]==0.27.0" --extra-index-url ${{ secrets.SQLMESH_ENTERPRISE_TOKEN }} | ||
shell: bash | ||
|
||
- name: Append raw events to tcloud_raw_data.raw_events to display observer metrics | ||
run: | | ||
python demo_scripts/main.py append-rawdata | ||
env: | ||
GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }} | ||
|
||
- name: Run SQLMesh | ||
run: | | ||
sqlmesh run | ||
env: | ||
TOBIKO_CLOUD_TOKEN: '${{ secrets.TOBIKO_CLOUD_TOKEN }}' | ||
GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }} |