scrape.yml

name: Run Scrapy Spiders

on:
  push:
    branches: [master]
  pull_request:
    branches: [master]
  schedule:
    - cron: "0 0 * * *" # Run daily at midnight UTC

jobs:
  scrape:
    permissions:
      contents: write
      pull-requests: write

    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.x"
          cache: "pip"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -r requirements.txt scrapy
          playwright install

      - name: Run scraping script
        run: ./scrape.sh

      - name: Move CSV files to public directory
        run: |
          mkdir -p public
          mv output/*.csv public/

      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          token: ${{ secrets.MEONG }}
          commit-message: Update scraped data
          title: Update scraped data
          body: |
            This PR updates the scraped data.
            - Auto-generated by [create-pull-request][1]

            [1]: https://github.com/peter-evans/create-pull-request
          branch: update-scraped-data
          base: master
          delete-branch: true

      - name: Upload CSV files as artifact
        uses: actions/upload-artifact@v3
        with:
          name: scraped-data
          path: public/*.csv