diff --git a/.github/workflows/el_ssa_ip_addresses.yml b/.github/workflows/el_ssa_ip_addresses.yml new file mode 100644 index 00000000..36596529 --- /dev/null +++ b/.github/workflows/el_ssa_ip_addresses.yml @@ -0,0 +1,31 @@ +name: Meltano - Spreadsheets Anywhere IP Addresses + +on: + schedule: + - cron: '0 10 * * *' + + workflow_dispatch: + +jobs: + el_ssa_ip_addresses: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=spreadsheets_anywhere_el + # Run Test + - run: meltano run spreadsheets_anywhere_el \ No newline at end of file diff --git a/.github/workflows/hub_metrics_publish.yml b/.github/workflows/hub_metrics_publish.yml new file mode 100644 index 00000000..5bf5c533 --- /dev/null +++ b/.github/workflows/hub_metrics_publish.yml @@ -0,0 +1,35 @@ +name: Meltano - Hub Metrics Publish + +on: + schedule: + - cron: '0 12 * * *'' + + workflow_dispatch: + +jobs: + hub_metrics_publish: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + TAP_SNOWFLAKE_METRICS_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS"]' + TAP_SNOWFLAKE_AUDIT_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.HUB_METRICS_AUDIT"]' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=hub_metrics_publish + # Run Test + - run: meltano run hub_metrics_publish \ No newline at end of file diff --git a/.github/workflows/slack_notifications.yml b/.github/workflows/slack_notifications.yml new file mode 100644 index 00000000..3b88730d --- /dev/null +++ b/.github/workflows/slack_notifications.yml @@ -0,0 +1,35 @@ +name: Meltano - Slack Publish + +on: + schedule: + - cron: '0 12 * * *' + + workflow_dispatch: + +jobs: + slack_notifications: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + TAP_SNOWFLAKE_SINGER_ACTIVITY: '["MELTANO_HUB.SINGER_ACTIVITY_NOTIFICATIONS"]' + TAP_SNOWFLAKE_MELTANO_ACTIVITY: '["MELTANO_HUB.MELTANO_ACTIVITY_NOTIFICATIONS"]' + TARGET_APPRISE_SINGER_ACTIVITY_URIS: ${{ secrets.APPRISE_SINGER_ACTIVITY_WEBHOOK }} + TARGET_APPRISE_MELTANO_ACTIVITY_URIS: ${{ secrets.APPRISE_MELTANO_ACTIVITY_WEBHOOK }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=slack_notifications + # Run Test + - run: meltano run slack_notifications \ No newline at end of file diff --git a/data/orchestrate/orchestrators.meltano.yml b/data/orchestrate/orchestrators.meltano.yml index 2f0c2ac1..88701aef 100644 --- a/data/orchestrate/orchestrators.meltano.yml +++ b/data/orchestrate/orchestrators.meltano.yml @@ -97,7 +97,7 @@ jobs: - name: slack_notifications tasks: - - dbt-snowflake:run_slack_notifications + # - dbt-snowflake:run_slack_notifications - tap-snowflake-singer-activity target-apprise-singer-activity - tap-snowflake-meltano-activity target-apprise-meltano-activity @@ -117,8 +117,8 @@ jobs: - name: spreadsheets_anywhere_el tasks: - tap-spreadsheets-anywhere coalesce-gcp-ips target-snowflake - - dbt-snowflake:test_source_spreadsheets_anywhere - - dbt-snowflake:run_snapshot_spreadsheets_anywhere + # - dbt-snowflake:test_source_spreadsheets_anywhere + # - dbt-snowflake:run_snapshot_spreadsheets_anywhere - name: snowplow_el tasks: @@ -160,8 +160,8 @@ jobs: - name: hub_metrics_publish tasks: - - dbt-snowflake:run_hub_metrics - - dbt-snowflake:test_hub_metrics + # - dbt-snowflake:run_hub_metrics + # - dbt-snowflake:test_hub_metrics # - great_expectations:test_dbt_hub_metrics - tap-snowflake-metrics target-yaml-metrics awscli:s3_copy_metrics - tap-snowflake-audit target-yaml-audit awscli:s3_copy_audit