crawl.yaml

name: "Generate extra info from index"
on:
  schedule:
    - cron: "0 */7 * * *"
  # push:
  #   branches:
  #     - 'main'
  #   paths:
  #     - 'index.json'
  workflow_dispatch:

env:
  PYTHONUNBUFFERED: 1 # ensure that python scripts live output instead of buffering

jobs:
  index-crawl:
    name: index-crawl
    runs-on: ubuntu-latest
    steps:

      - name: Checkout
        uses: actions/checkout@v4

      - uses: actions/setup-python@v4
        with:
          cache: 'pip'

      - run: pip install -r requirements.txt
        working-directory: scripts

      - run: python platforms.py
        working-directory: scripts

      - run: python index_to_letters.py
        working-directory: scripts
      
      - run: python gen-alt-index.py
        working-directory: scripts
      
      - run: python scrape.py
        working-directory: scripts
      
      - run: python remove_dead_links.py
        working-directory: scripts

      - name: Commit changes
        if: always()
        run: |
          set -eux

          git config --local user.email "helper@utf9k.net"
          git config --local user.name "Indexer Bot"
          git config pull.rebase false

          git pull
          git add .
          git commit -a -m "Index extras have been updated" || exit 0
      
      - name: Push changes
        if: always()
        uses: ad-m/github-push-action@master
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: ${{ github.ref }}

      - name: Sync index to API
        if: always()
        uses: up9cloud/action-rsync@master
        env:
          USER: gharunner
          HOST: khindex.utf9k.net
          KEY: ${{ secrets.DEPLOY_SSH_KEY }}
          SOURCE: index-alt.json
          TARGET: /var/www/html/khindex/index.json
          ARGS: -azv
    
      - name: Sync alphabetical groupings to API
        if: always()
        uses: up9cloud/action-rsync@master
        env:
          USER: gharunner
          HOST: khindex.utf9k.net
          KEY: ${{ secrets.DEPLOY_SSH_KEY }}
          SOURCE: letters/
          TARGET: /var/www/html/khindex/letters
          ARGS: -azv

      - name: Sync albums to API
        if: always()
        uses: up9cloud/action-rsync@master
        env:
          USER: gharunner
          HOST: khindex.utf9k.net
          KEY: ${{ secrets.DEPLOY_SSH_KEY }}
          SOURCE: albums/
          TARGET: /var/www/html/khindex/albums
          ARGS: -azv
        
      - name: Sync platform list to API
        if: always()
        uses: up9cloud/action-rsync@master
        env:
          USER: gharunner
          HOST: khindex.utf9k.net
          KEY: ${{ secrets.DEPLOY_SSH_KEY }}
          SOURCE: platforms.json
          TARGET: /var/www/html/khindex/platforms.json
          ARGS: -azv