add ci/cd to google sheets #13
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: id-jobs-update | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request_target: | |
| schedule: | |
| - cron: "0 0 * * *" # Schedule to run at midnight (UTC time) | |
| workflow_dispatch: # Add this line to enable manual triggering | |
| jobs: | |
| id-jobs-update: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.MEONG }} | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.head_ref }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.x" | |
| cache: "pip" | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Create pip cache directory | |
| run: mkdir -p ~/.cache/pip | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt scrapy | |
| playwright install | |
| env: | |
| PIP_CACHE_DIR: .pip_cache | |
| - name: Validate Python installation | |
| run: | | |
| python --version | |
| pip --version | |
| continue-on-error: false | |
| - name: Run scraping script | |
| run: ./scrape.sh | |
| - name: Check for CSV files | |
| run: | | |
| if ls output/*.csv 1> /dev/null 2>&1; then | |
| echo "CSV files found." | |
| else | |
| echo "No CSV files found. Exiting." | |
| exit 1 | |
| fi | |
| - name: Move CSV files to public directory | |
| run: | | |
| mkdir -p public | |
| mv output/*.csv public/ | |
| - name: Create Pull Request if changes are detected | |
| uses: stefanzweifel/git-auto-commit-action@v5 | |
| with: | |
| commit_message: "[skip ci] - updated" | |
| commit_user_name: "github-actions[bot]" | |
| commit_user_email: "github-actions[bot]@users.noreply.github.com" | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.MEONG }} | |
| - name: Set up Google Cloud SDK | |
| uses: google-github-actions/setup-gcloud@v1 | |
| with: | |
| project_id: ${{ secrets.GCP_PROJECT_ID }} | |
| service_account_key: ${{ secrets.GCP_SA_KEY }} | |
| export_default_credentials: true | |
| - name: Install Google Sheets API client library | |
| run: pip install google-auth google-auth-oauthlib google-auth-httplib2 google-api-python-client | |
| - name: Update Google Sheets | |
| env: | |
| SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }} | |
| run: python update_sheets.py | |
| - name: Upload CSV files as artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: scraped-data | |
| path: public/*.csv |