schedules for non-arch jobs

meltano · Aug 5, 2024 · f065636 · f065636
1 parent c0a069d
commit f065636
Show file tree

Hide file tree

Showing 4 changed files with 106 additions and 5 deletions.
diff --git a/.github/workflows/el_ssa_ip_addresses.yml b/.github/workflows/el_ssa_ip_addresses.yml
@@ -0,0 +1,31 @@
+name: Meltano - Spreadsheets Anywhere IP Addresses
+
+on:
+  schedule:
+    - cron:  '0 10 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  el_ssa_ip_addresses:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=spreadsheets_anywhere_el
+    # Run Test
+    - run: meltano run spreadsheets_anywhere_el
diff --git a/.github/workflows/hub_metrics_publish.yml b/.github/workflows/hub_metrics_publish.yml
@@ -0,0 +1,35 @@
+name: Meltano - Hub Metrics Publish
+
+on:
+  schedule:
+    - cron:  '0 12 * * *''
+
+  workflow_dispatch:
+
+jobs:
+  hub_metrics_publish:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      TAP_SNOWFLAKE_METRICS_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS"]'
+      TAP_SNOWFLAKE_AUDIT_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.HUB_METRICS_AUDIT"]'
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=hub_metrics_publish
+    # Run Test
+    - run: meltano run hub_metrics_publish
diff --git a/.github/workflows/slack_notifications.yml b/.github/workflows/slack_notifications.yml
@@ -0,0 +1,35 @@
+name: Meltano - Slack Publish
+
+on:
+  schedule:
+    - cron:  '0 12 * * *'
+
+  workflow_dispatch:
+
+jobs:
+  slack_notifications:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: data/
+    environment: prod
+    env:
+      MELTANO_ENVIRONMENT: prod
+      MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false'
+      SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
+      TAP_SNOWFLAKE_SINGER_ACTIVITY: '["MELTANO_HUB.SINGER_ACTIVITY_NOTIFICATIONS"]'
+      TAP_SNOWFLAKE_MELTANO_ACTIVITY: '["MELTANO_HUB.MELTANO_ACTIVITY_NOTIFICATIONS"]'
+      TARGET_APPRISE_SINGER_ACTIVITY_URIS: ${{ secrets.APPRISE_SINGER_ACTIVITY_WEBHOOK }}
+      TARGET_APPRISE_MELTANO_ACTIVITY_URIS: ${{ secrets.APPRISE_MELTANO_ACTIVITY_WEBHOOK }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+    - run: pip install -r requirements.txt
+    - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env
+    # Install Plugins
+    - run: meltano install --schedule=slack_notifications
+    # Run Test
+    - run: meltano run slack_notifications
diff --git a/data/orchestrate/orchestrators.meltano.yml b/data/orchestrate/orchestrators.meltano.yml
@@ -97,7 +97,7 @@ jobs:
 
 - name: slack_notifications
   tasks:
-  - dbt-snowflake:run_slack_notifications
+  # - dbt-snowflake:run_slack_notifications
   - tap-snowflake-singer-activity target-apprise-singer-activity
   - tap-snowflake-meltano-activity target-apprise-meltano-activity
 
@@ -117,8 +117,8 @@ jobs:
 - name: spreadsheets_anywhere_el
   tasks:
   - tap-spreadsheets-anywhere coalesce-gcp-ips target-snowflake
-  - dbt-snowflake:test_source_spreadsheets_anywhere
-  - dbt-snowflake:run_snapshot_spreadsheets_anywhere
+  # - dbt-snowflake:test_source_spreadsheets_anywhere
+  # - dbt-snowflake:run_snapshot_spreadsheets_anywhere
 
 - name: snowplow_el
   tasks:
@@ -160,8 +160,8 @@ jobs:
 
 - name: hub_metrics_publish
   tasks:
-  - dbt-snowflake:run_hub_metrics
-  - dbt-snowflake:test_hub_metrics
+  # - dbt-snowflake:run_hub_metrics
+  # - dbt-snowflake:test_hub_metrics
   # - great_expectations:test_dbt_hub_metrics
   - tap-snowflake-metrics target-yaml-metrics awscli:s3_copy_metrics
   - tap-snowflake-audit target-yaml-audit awscli:s3_copy_audit