Python script modified for business and individual review #35

Workflow file for this run

.github/workflows/project.yml at e945d5b


	# name: project.py

	# on:
	# schedule:
	# - cron: '0 0 * * 5' # At 00:00 on Friday
	# push:
	# branches:
	# - main

	# jobs:
	# run_Mealdb:
	# runs-on: ubuntu-latest
	# outputs:
	# script_name: 'Mealdb Script'
	# status: ${{ steps.set_status.outputs.status }}
	# error: ${{ steps.set_status.outputs.error }}
	# steps:
	# - uses: actions/checkout@v2

	# - uses: actions/setup-python@v4
	# with:
	# python-version: '3.10'

	# - uses: actions/cache@v3
	# with:
	# path: ~/.cache/pip
	# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	# restore-keys: \|
	# ${{ runner.os }}-pip-

	# - name: Install dependencies
	# run: \|
	# python -m pip install --upgrade pip
	# pip install -r requirements.txt

	# - name: Execute mealdb Scrapy spider
	# id: mealdb_execution
	# run: \|
	# scrapy crawl mealdb -O ./data/raw/mealdb.json . 2>mealdb_error.log \|\| echo "error" > mealdb_error.log
	# working-directory: ./landing_zone/collectors/Mealdb/mealscraper

	# - name: Set job status and error
	# id: set_status
	# if: always()
	# run: \|
	# if [ -s mealdb_error.log ]; then
	# ERROR_MSG=$(<mealdb_error.log)
	# echo "::set-output name=status::failure"
	# echo "::set-output name=error::$ERROR_MSG"
	# else
	# echo "::set-output name=status::success"
	# echo "::set-output name=error::None"
	# fi

	# run_ApprovedFood:
	# runs-on: ubuntu-latest
	# outputs:
	# script_name: 'ApprovedFood Script'
	# status: ${{ steps.set_status.outputs.status }}
	# error: ${{ steps.set_status.outputs.error }}
	# steps:
	# - uses: actions/checkout@v2
	# - uses: actions/setup-python@v4
	# with:
	# python-version: '3.10'
	# - uses: actions/cache@v3
	# with:
	# path: ~/.cache/pip
	# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	# - name: Install dependencies
	# run: \|
	# python -m pip install --upgrade pip
	# pip install -r requirements.txt
	# - name: Execute approvedfood Scrapy spider
	# run: \|
	# scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json 2>approvedfood_error.log \|\| echo "error" > approvedfood_error.log
	# working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper
	# - name: Set job status and error
	# id: set_status
	# if: always()
	# run: \|
	# if [ -s approvedfood_error.log ]; then
	# ERROR_MSG=$(<approvedfood_error.log)
	# echo "::set-output name=status::failure"
	# echo "::set-output name=error::$ERROR_MSG"
	# else
	# echo "::set-output name=status::success"
	# echo "::set-output name=error::None"
	# fi

	# run_eat_by_date:
	# runs-on: ubuntu-latest
	# outputs:
	# script_name: 'Eat by Date Script'
	# status: ${{ steps.set_status.outputs.status }}
	# error: ${{ steps.set_status.outputs.error }}
	# steps:
	# - uses: actions/checkout@v2
	# - uses: actions/setup-python@v4
	# with:
	# python-version: '3.10'
	# - uses: actions/cache@v3
	# with:
	# path: ~/.cache/pip
	# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	# - name: Install dependencies
	# run: \|
	# python -m pip install --upgrade pip
	# pip install -r requirements.txt
	# - name: Execute eat_by_date.py script
	# run: \|
	# python eat_by_date.py 2>eat_by_date_error.log \|\| echo "error" > eat_by_date_error.log
	# working-directory: ./landing_zone/collectors/eat_by_date
	# - name: Set job status and error
	# id: set_status
	# if: always()
	# run: \|
	# if [ -s eat_by_date_error.log ]; then
	# ERROR_MSG=$(<eat_by_date_error.log)
	# echo "::set-output name=status::failure"
	# echo "::set-output name=error::$ERROR_MSG"
	# else
	# echo "::set-output name=status::success"
	# echo "::set-output name=error::None"
	# fi

	# notify_slack:
	# needs: [run_Mealdb, run_ApprovedFood] #, run_eat_by_date]
	# runs-on: ubuntu-latest
	# steps:
	# - name: Set current date
	# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

	# - name: Notify Slack on Error
	# uses: 8398a7/action-slack@v3
	# with:
	# status: custom
	# custom_payload: \|
	# {
	# "attachments": [{
	# "color": "${{ needs.run_Mealdb.outputs.status }} === 'success' && '${{ needs.run_ApprovedFood.outputs.status }} === 'success' ? 'good' : 'danger'",
	# "text": "Workflow: ${{ github.workflow }}\nDate: ${{ env.CURRENT_DATE }}\nJob: Detailed Status Update\n- Mealdb Script Status: ${{ needs.run_Mealdb.outputs.status }} (Error: ${{ needs.run_Mealdb.outputs.error }})\n- ApprovedFood Script Status: ${{ needs.run_ApprovedFood.outputs.status }} (Error: ${{ needs.run_ApprovedFood.outputs.error }})\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
	# }]
	# }
	# env:
	# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
	# CURRENT_DATE: ${{ env.CURRENT_DATE }}
	# if: always()



	name: project.py

	on:
	schedule:
	- cron: '0 0 * * 5' # At 00:00 on Friday
	push:
	branches:
	- main

	jobs:
	run_Mealdb:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Cache Python packages
	uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-

	- name: Install dependencies
	run: python -m pip install --upgrade pip && pip install -r requirements.txt

	- name: Execute mealdb Scrapy spider
	run: scrapy crawl mealdb -O ./data/raw/mealdb.json
	working-directory: ./landing_zone/collectors/Mealdb/mealscraper
	continue-on-error: true

	- name: Prepare error log
	id: prep_log
	if: failure()
	run: \|
	echo "ERROR_LOG<<EOF" >> $GITHUB_ENV
	cat mealdb_error.log >> $GITHUB_ENV
	echo "EOF" >> $GITHUB_ENV
	working-directory: ./landing_zone/collectors/Mealdb/mealscraper

	- name: Determine success/failure indicator and message (Mealdb)
	id: status_indicator
	run: \|
	if [ -s mealdb_error.log ]; then
	echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
	echo "COLOR=#ff0000" >> $GITHUB_ENV
	else
	echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
	echo "COLOR=#36a64f" >> $GITHUB_ENV
	fi
	working-directory: ./landing_zone/collectors/Mealdb/mealscraper


	- name: Set current date
	run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

	- name: Notify Slack (Mealdb Script)
	if: always()
	uses: 8398a7/action-slack@v3
	with:
	status: custom
	custom_payload: \|
	{
	"attachments": [
	{
	"color": "${{ env.COLOR }}",
	"text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
	}
	]
	}
	env:
	SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

	run_ApprovedFood:
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@v2

	- name: Setup Python
	uses: actions/setup-python@v4
	with:
	python-version: '3.10'

	- name: Cache Python packages
	uses: actions/cache@v3
	with:
	path: ~/.cache/pip
	key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	restore-keys: \|
	${{ runner.os }}-pip-

	- name: Install dependencies
	run: python -m pip install --upgrade pip && pip install -r requirements.txt

	- name: Execute approvedfood Scrapy spider
	run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json
	working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper
	continue-on-error: true

	- name: Prepare error log (ApprovedFood)
	if: failure()
	run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
	working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper

	- name: Determine success/failure indicator and message (Mealdb)
	id: status_indicator
	run: \|
	if [ -s mealdb_error.log ]; then
	echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
	echo "COLOR=#ff0000" >> $GITHUB_ENV
	else
	echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
	echo "COLOR=#36a64f" >> $GITHUB_ENV
	fi
	working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper


	- name: Set current date (ApprovedFood)
	run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

	- name: Notify Slack (ApprovedFood Script)
	if: always()
	uses: 8398a7/action-slack@v3
	with:
	status: custom
	custom_payload: \|
	{
	"attachments": [
	{
	"color": "${{ env.COLOR }}",
	"text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
	}
	]
	}
	env:
	SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

	# run_eat_by_date:
	# runs-on: ubuntu-latest
	# steps:
	# - uses: actions/checkout@v2

	# - name: Setup Python
	# uses: actions/setup-python@v4
	# with:
	# python-version: '3.10'

	# - name: Cache Python packages
	# uses: actions/cache@v3
	# with:
	# path: ~/.cache/pip
	# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
	# restore-keys: \|
	# ${{ runner.os }}-pip-

	# - name: Install dependencies
	# run: python -m pip install --upgrade pip && pip install -r requirements.txt

	# - name: Execute eat_by_date.py script
	# run: python eat_by_date.py
	# working-directory: ./landing_zone/collectors/eat_by_date
	# continue-on-error: true

	# - name: Prepare error log (Eat by Date)
	# if: failure()
	# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
	# working-directory: ./landing_zone/collectors/eat_by_date

	# - name: Set current date (Eat by Date)
	# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

	# - name: Notify Slack (Eat by Date Script)
	# if: always()
	# uses: 8398a7/action-slack@v3
	# with:
	# status: custom
	# custom_payload: \|
	# {
	# "attachments": [
	# {
	# "color": "${{ job.status }} == 'success' ? 'good' : 'danger'",
	# "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
	# }
	# ]
	# }
	# env:
	# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Python script modified for business and individual review #35

Workflow file

Python script modified for business and individual review #35

Jobs

Run details

Workflow file for this run