This repository has been archived by the owner on May 6, 2024. It is now read-only.
Validate CSV #12710
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Validate CSV | |
| on: | |
| schedule: | |
| - cron: '15 */4 * * *' | |
| workflow_dispatch: ~ | |
| push: | |
| branches: [ master ] | |
| paths: | |
| - '**.csv' | |
| pull_request: | |
| branches: [ master ] | |
| paths: | |
| - '**.csv' | |
| jobs: | |
| validate: | |
| runs-on: ubuntu-20.04 | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python 3.7 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.7 | |
| - name: Install dependencies | |
| run: | | |
| npm ci | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Validate structure and content of CSVs | |
| run: node scripts/validate-csv.js fallzahlen_kanton_total_csv_v2/*.csv | |
| - name: Check if there are empty lines | |
| run: scripts/check_for_empty_lines.sh fallzahlen_kanton_total_csv_v2/*.csv | |
| - name: Check for outliers in CSVs | |
| run: python scripts/check_for_outliers.py fallzahlen_kanton_total_csv_v2/*.csv | |
| - name: Get current unix timestamp | |
| if: always() | |
| id: date | |
| run: echo "ts=$(date +'%s')" >> $GITHUB_OUTPUT | |
| # notify slack if a CSV validation failed | |
| - name: Notify slack failure | |
| if: ${{ failure() }} | |
| env: | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| uses: pullreminders/slack-action@master | |
| with: | |
| args: '{\"channel\":\"C013C0UUQ4S\", \"attachments\": [{\"fallback\": \"Job failed.\", \"color\": \"danger\", \"title\": \"Validate CSV\", \"title_link\": \"https://github.com/openZH/covid_19/actions/runs/${{ github.run_id }}?check_suite_focus=true\", \"text\": \":x: CSV validation failed\", \"footer\": \"<https://github.com/openZH/covid_19|openZH/covid_19>\", \"footer_icon\": \"https://github.com/abinoda/slack-action/raw/master/docs/app-icons/github-icon.png\", \"ts\": \"${{steps.date.outputs.ts}}\"}]}' |