🧃 Daily ELT Job 🧃 #60
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily ELT Job | |
| run-name: 🧃 Daily ELT Job 🧃 | |
| on: | |
| schedule: | |
| - cron: '0 14 * * 1-5' # 7am PST (14:00 UTC) Monday to Friday | |
| workflow_dispatch: # Allow manual triggering | |
| jobs: | |
| daily_elt: | |
| name: Daily ELT Process | |
| runs-on: ubuntu-latest | |
| if: github.ref == 'refs/heads/main' # Only run on the main branch | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Setup Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12' | |
| - name: Authenticate to Google Cloud | |
| uses: 'google-github-actions/auth@v2' | |
| with: | |
| credentials_json: '${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}' | |
| - name: Install SQLMesh Enterprise + Dependencies | |
| run: | | |
| pip install poetry keyring keyrings-google-artifactregistry-auth | |
| poetry install --no-root | |
| shell: bash | |
| - name: Append raw events to tcloud_raw_data.raw_events to display observer metrics | |
| run: | | |
| poetry run python demo_scripts/main.py append-rawdata | |
| env: | |
| GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }} | |
| - name: Run SQLMesh | |
| run: | | |
| poetry run tcloud sqlmesh run | |
| env: | |
| TCLOUD_TOKEN: ${{ secrets.TCLOUD_TOKEN }} | |
| GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }} |