Daily Job Data Update #10

	name: id-jobs-update

	on:
	push:
	branches:
	- main
	pull_request_target:

	schedule:
	- cron: "0 0 * * *" # Schedule to run at midnight (UTC time)

	workflow_dispatch: # Add this line to enable manual triggering

	jobs:
	id-jobs-update:
	runs-on: ubuntu-latest

	permissions:
	contents: write

	steps:
	- name: Checkout code
	uses: actions/checkout@v4
	with:
	token: ${{ secrets.MEONG }}
	repository: ${{ github.event.pull_request.head.repo.full_name }}
	ref: ${{ github.head_ref }}

	- name: Set up Python
	uses: actions/setup-python@v4
	with:
	python-version: "3.x"
	cache: "pip"

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt scrapy
	playwright install
	env:
	PIP_CACHE_DIR: .pip_cache

	- name: Validate Python installation
	run: \|
	python --version
	pip --version
	continue-on-error: false

	- name: Run scraping script
	run: ./scrape.sh

	- name: Check for CSV files
	run: \|
	if ls output/*.csv 1> /dev/null 2>&1; then
	echo "CSV files found."
	else
	echo "No CSV files found. Exiting."
	exit 1
	fi

	- name: Move CSV files to public directory
	run: \|
	mkdir -p public
	mv output/*.csv public/

	- name: Create Pull Request if changes are detected
	uses: stefanzweifel/git-auto-commit-action@v5
	with:
	commit_message: "[skip ci] - updated"
	commit_user_name: "github-actions[bot]"
	commit_user_email: "github-actions[bot]@users.noreply.github.com"
	env:
	GITHUB_TOKEN: ${{ secrets.MEONG }}

	- name: Upload CSV files as artifact
	uses: actions/upload-artifact@v3
	with:
	name: scraped-data
	path: public/*.csv

Provide feedback