Skip to content

Log added

Log added #101

Workflow file for this run

# name: project.py
# on:
# schedule:
# - cron: '0 0 * * 5' # At 00:00 on Friday
# push:
# branches:
# - main
# jobs:
# run_Mealdb:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute mealdb Scrapy spider
# run: scrapy crawl mealdb -O ./data/raw/mealdb.json
# working-directory: ./landing_zone/collectors/meal_db/mealscraper
# continue-on-error: true
# - name: Prepare error log
# id: prep_log
# if: failure()
# run: |
# echo "ERROR_LOG<<EOF" >> $GITHUB_ENV
# cat mealdb_error.log >> $GITHUB_ENV
# echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/meal_db/mealscraper
# - name: Determine success/failure indicator and message (Mealdb)
# id: status_indicator
# run: |
# if [ -s mealdb_error.log ]; then
# echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
# echo "COLOR=#ff0000" >> $GITHUB_ENV
# else
# echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
# echo "COLOR=#36a64f" >> $GITHUB_ENV
# fi
# working-directory: ./landing_zone/collectors/meal_db/mealscraper
# - name: Set current date
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack (Mealdb Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ env.COLOR }}",
# "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# run_ApprovedFood:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute approvedfood Scrapy spider
# run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json
# working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
# continue-on-error: true
# - name: Prepare error log (ApprovedFood)
# if: failure()
# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
# - name: Determine success/failure indicator and message (Mealdb)
# id: status_indicator
# run: |
# if [ -s mealdb_error.log ]; then
# echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
# echo "COLOR=#ff0000" >> $GITHUB_ENV
# else
# echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
# echo "COLOR=#36a64f" >> $GITHUB_ENV
# fi
# working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
# - name: Set current date (ApprovedFood)
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack (ApprovedFood Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ env.COLOR }}",
# "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# run_eat_by_date:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute eat_by_date.py script
# run: python eat_by_date.py
# working-directory: ./landing_zone/collectors/eat_by_date
# continue-on-error: true
# - name: Prepare error log (Eat by Date)
# if: failure()
# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/eat_by_date
# - name: Set current date (Eat by Date)
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack (Eat by Date Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ job.status }} == 'success' ? 'good' : 'danger'",
# "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
name: project.py
on:
schedule:
- cron: '0 0 * * 5' # At 00:00 on Friday
push:
branches:
- main
jobs:
run_Mealdb:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute mealdb Scrapy spider
run: scrapy crawl mealdb -O ./data/raw/mealdb.json || echo "Script failed, check logs." > mealdb_error.log
working-directory: ./landing_zone/collectors/meal_db/mealscraper
# continue-on-error: false
- name: Upload error log to Gist (mealdb)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat mealdb_error.log)
JSON_PAYLOAD="{\"description\": \"Error Log for mealdb\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (Mealdb)
id: status_indicator
run: |
if [ -s mealdb_error.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/collectors/meal_db/mealscraper
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (Mealdb Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: <$GIST_URL|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_ApprovedFood:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute approvedfood Scrapy spider
run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json || echo "Script failed, check logs." > approvedfood_error.log
working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
# continue-on-error: false
- name: Prepare error log (ApprovedFood)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
- name: Determine success/failure indicator and message (Mealdb)
id: status_indicator
run: |
if [ -s approvedfood_error.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Upload error log to Gist (appprovedfood)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat approvedfood_error.log)
JSON_PAYLOAD="{\"description\": \"Error Log for Approvedfood\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Set current date (ApprovedFood)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (ApprovedFood Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View URL>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# run_eat_by_date:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute eat_by_date.py script
# run: python eat_by_date.py || echo "Script failed, check logs." > eat_by_date_error.log
# working-directory: ./landing_zone/collectors/eat_by_date
# continue-on-error: true
# - name: Prepare error log (Eat by Date)
# if: failure()
# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/eat_by_date
# - name: Upload error log to Gist (Eat by Date)
# if: failure()
# id: upload_log
# run: |
# LOG_CONTENT=$(cat eat_by_date.log)
# JSON_PAYLOAD="{\"description\": \"Error Log for Eat by date\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
# GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
# echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
# - name: Set current date (Eat by Date)
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Determine Color
# id: set_color
# run: |
# if [ "${{ job.status }}" == "success" ]; then
# echo "::set-output name=color::#36a64f"
# else
# echo "::set-output name=color::#ff0000"
# fi
# - name: Notify Slack (Eat by Date Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ steps.set_color.outputs.color }}",
# "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
# run_flipkart:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute scrap_flipkart_pages script
# run: python scrap_flipkart_pages_sel.py || echo "Script failed, check logs." > flipkart_error.log
# working-directory: ./landing_zone/collectors/Flipkart
# continue-on-error: true
# - name: Prepare error log (flipkart)
# if: failure()
# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && flipkart_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/Flipkart
# - name: Upload error log to Gist (flipkart)
# if: failure()
# id: upload_log
# run: |
# LOG_CONTENT=$(cat flipkart_error.log)
# JSON_PAYLOAD="{\"description\": \"Error Log for flipkat\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
# GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
# echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
# - name: Determine success/failure indicator and message (Mealdb)
# id: status_indicator
# run: |
# if [ -s flipkart_error.log ]; then
# echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
# echo "COLOR=#ff0000" >> $GITHUB_ENV
# else
# echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
# echo "COLOR=#36a64f" >> $GITHUB_ENV
# fi
# working-directory: ./landing_zone/collectors/Flipkart
# - name: Determine Color
# id: set_color
# run: |
# if [ "${{ job.status }}" == "success" ]; then
# echo "::set-output name=color::#36a64f"
# else
# echo "::set-output name=color::#ff0000"
# fi
# - name: Set current date (flipkart)
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack (flipkart Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ steps.set_color.outputs.color }}",
# "text": "Flipkart Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_big_basket:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute bigBasket script
run: python big_basket.py || echo "Script failed, check logs." > bigBasket.log
working-directory: ./landing_zone/collectors/big_basket
# continue-on-error: false
- name: Prepare error log (bigBasket)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat bigBasket.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/collectors/big_basket
- name: Upload error log to Gist (bigbasket)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat bigBasket.log)
JSON_PAYLOAD="{\"description\": \"Error Log for Bigbasket\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (bigBasket)
id: status_indicator
run: |
if [ -s bigBasket.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/collectors/big_basket
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (bigBasket)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (bigBasket Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "BigBasket Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# run_catalonia_establishment_locationt:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute catalonia_establishment_location script
# run: python establishments_scraper.py || echo "Script failed, check logs." > catalonia_establishment_location.log
# working-directory: ./landing_zone/collectors/catalonia_establishment_location
# continue-on-error: true
# - name: Prepare error log (establishment location)
# if: failure()
# run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat catalonia_establishment_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
# working-directory: ./landing_zone/collectors/catalonia_establishment_location
# - name: Upload error log to Gist (Establishment Location)
# if: failure()
# id: upload_log
# run: |
# LOG_CONTENT=$(cat catalonia_establishment_location.log)
# JSON_PAYLOAD="{\"description\": \"Error Log for Establishment Location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
# GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
# echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
# - name: Determine success/failure indicator and message (catalonia_establishment_location)
# id: status_indicator
# run: |
# if [ -s catalonia_establishment_location.log ]; then
# echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
# echo "COLOR=#ff0000" >> $GITHUB_ENV
# else
# echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
# echo "COLOR=#36a64f" >> $GITHUB_ENV
# fi
# working-directory: ./landing_zone/collectors/catalonia_establishment_location
# - name: Determine Color
# id: set_color
# run: |
# if [ "${{ job.status }}" == "success" ]; then
# echo "::set-output name=color::#36a64f"
# else
# echo "::set-output name=color::#ff0000"
# fi
# - name: Set current date (catalonia_establishment_location)
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack (catalonia_establishment_location Script)
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ steps.set_color.outputs.color }}",
# "text": "Catalonia Establishment Location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_OCR:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute OCR script
run: python ocr_marcadona.py || echo "Script failed, check logs." > ocr.log
working-directory: ./landing_zone/collectors/OCR
# continue-on-error: false
- name: Prepare error log (OCR)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat ocr.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/collectors/OCR
- name: Upload error log to Gist (ocr)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat ocr.log)
JSON_PAYLOAD="{\"description\": \"Error Log for ocr\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (OCR)
id: status_indicator
run: |
if [ -s OCR.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/collectors/OCR
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (OCR)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (OCR Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "OCR Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_customers:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute customer script
run: python sm_retail_customer.py || echo "Script failed, check logs." > customer.log
working-directory: ./landing_zone/collectors/customers
# continue-on-error: false
- name: Prepare error log (customers)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customers.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/collectors/customers
- name: Upload error log to Gist (customers)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat customers.log)
JSON_PAYLOAD="{\"description\": \"Error Log for customers\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (customers)
id: status_indicator
run: |
if [ -s customers.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/collectors/customers
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (customers)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (customers Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Customers Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_customer_location:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute customer_location script
run: python customer_location.py || echo "Script failed, check logs." > customer_location.log
working-directory: ./landing_zone/synthetic/customer_location
# continue-on-error: false
- name: Prepare error log (customer_location)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/synthetic/customer_location
- name: Upload error log to Gist (customer location)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat customer_location.log)
JSON_PAYLOAD="{\"description\": \"Error Log for customer location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (customer_location)
id: status_indicator
run: |
if [ -s customer_location.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/synthetic/customer_location
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (customer_location)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (customer_location Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Customer location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_customer_purchase:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute customer_purchase script
run: python customer_purchase.py || echo "Script failed, check logs." > customer_purchase.log
working-directory: ./landing_zone/synthetic/customer_purchase
# continue-on-error: false
- name: Prepare error log (customer_purchase)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_purchase.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/synthetic/customer_purchase
- name: Upload error log to Gist (customer purchase)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat customer_purchase.log)
JSON_PAYLOAD="{\"description\": \"Error Log for customer purchase\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (customer_purchase)
id: status_indicator
run: |
if [ -s customer_purchase.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/synthetic/customer_purchase
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (customer_purchase)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (customer_purchase Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Customer purchase Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_sentiment_reviews:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute sentiment reviews script
run: python business_reviews.py || echo "Script failed, check logs." > sentiment_reviews.log
working-directory: ./landing_zone/synthetic/sentiment reviews
# continue-on-error: false
- name: Execute sentiment reviews script
run: python individual_reviews.py || echo "Script failed, check logs." > sentiment_reviews.log
working-directory: ./landing_zone/synthetic/sentiment reviews
# continue-on-error: false
- name: Prepare error log (sentiment reviews)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat sentiment_reviews.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/synthetic/sentiment reviews
- name: Upload error log to Gist (sentiment_reviews)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat sentiment_reviews.log)
JSON_PAYLOAD="{\"description\": \"Error Log for sentiment reviews\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (sentiment reviews)
id: status_indicator
run: |
if [ -s sentiment_reviews.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/synthetic/sentiment reviews
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (sentiment reviews)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (sentiment reviews Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Sentiment reviews Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
run_supermarket_products:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: '3.10'
- name: Cache Python packages
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: python -m pip install --upgrade pip && pip install -r requirements.txt
- name: Execute supermarket_products script
run: python supermarket_products.py || echo "Script failed, check logs." > supermarket_products.log
working-directory: ./landing_zone/synthetic/supermarket_products
# continue-on-error: false
- name: Prepare error log (supermarket_products)
if: failure()
run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat supermarket_products.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
working-directory: ./landing_zone/synthetic/supermarket_products
- name: Upload error log to Gist (supermarket_products)
if: failure()
id: upload_log
run: |
LOG_CONTENT=$(cat supermarket_products.log)
JSON_PAYLOAD="{\"description\": \"Error Log for supermarket products\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
- name: Determine success/failure indicator and message (supermarket_products)
id: status_indicator
run: |
if [ -s supermarket_products.log ]; then
echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
echo "COLOR=#ff0000" >> $GITHUB_ENV
else
echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
echo "COLOR=#36a64f" >> $GITHUB_ENV
fi
working-directory: ./landing_zone/synthetic/supermarket_products
- name: Determine Color
id: set_color
run: |
if [ "${{ job.status }}" == "success" ]; then
echo "::set-output name=color::#36a64f"
else
echo "::set-output name=color::#ff0000"
fi
- name: Set current date (supermarket_products)
run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
- name: Notify Slack (supermarket_products Script)
if: always()
uses: 8398a7/action-slack@v3
with:
status: custom
custom_payload: |
{
"attachments": [
{
"color": "${{ steps.set_color.outputs.color }}",
"text": "Supermarket products Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
}
]
}
env:
SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# name: project.py
# on:
# schedule:
# - cron: '0 0 * * 5' # At 00:00 on Friday
# push:
# branches:
# - main
# jobs:
# run_scripts:
# runs-on: ubuntu-latest
# strategy:
# matrix:
# include:
# - name: 'ocr'
# path: 'OCR'
# command: 'python ocr_marcadona.py'
# - name: 'customers'
# path: 'customers'
# command: 'python sm_retail_customer.py'
# # - name: 'catalonia_establishment_location'
# # path: 'catalonia_establishment_location'
# # command: 'python establishments_scraper.py'
# # - name: 'flipkart'
# # path: 'Flipkart'
# # command: 'python scrap_flipkart_pages_sel.py'
# # - name: 'Eatbydate'
# # path: 'eat_by_date'
# # command: 'python eat_by_date.py'
# - name: 'Mealdb'
# path: 'meal_db/mealscraper'
# command: 'scrapy crawl mealdb -O ./data/raw/mealdb.json'
# - name: 'ApprovedFood'
# path: 'approved_food_uk/approvedfood_scraper'
# command: 'scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json'
# - name: 'BigBasket'
# path: 'big_basket'
# command: 'python big_basket.py'
# - name: 'CustomerLocation'
# path: 'synthetic/customer_location'
# command: 'python customer_location.py'
# - name: 'CustomerPurchase'
# path: 'synthetic/customer_purchase'
# command: 'python customer_purchase.py'
# - name: 'Business_SentimentReview'
# path: 'synthetic/sentiment reviews'
# command: 'python business_reviews.py'
# - name: 'Individual_SentimentReview'
# path: 'synthetic/sentiment reviews'
# command: 'python individual_reviews.py'
# - name: 'SupermarketProducts'
# path: 'synthetic/supermarket_products'
# command: 'python supermarket_products.py'
# steps:
# - uses: actions/checkout@v2
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - name: Cache Python packages
# uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: ${{ runner.os }}-pip-
# - name: Install dependencies
# run: python -m pip install --upgrade pip && pip install -r requirements.txt
# - name: Execute Script
# run: |
# cd ./landing_zone/collectors/${{ matrix.path }}
# ${{ matrix.command }}
# continue-on-error: true
# - name: Determine Color
# id: set color
# run: |
# if [ "${{ job.status }}" == "success" ]; then
# echo "::set-output name=color::#36a64f"
# else
# echo "::set-output name=color::#ff0000"
# fi
# - name: Prepare error log
# if: failure()
# run: |
# echo "ERROR_LOG<<EOF" >> $GITHUB_ENV
# cat ./logs/${{ matrix.name }}_error.log >> $GITHUB_ENV
# echo "EOF" >> $GITHUB_ENV
# - name: Upload error log to Gist
# if: failure()
# run: |
# LOG_CONTENT=$(cat ./logs/${{ matrix.name }}_error.log)
# JSON_PAYLOAD="{\"description\": \"Error Log for ${{ matrix.name }}\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
# GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
# echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV
# - name: Notify Slack
# if: always()
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [
# {
# "color": "${{ steps.set_color.outputs.color }}",
# "text": "${{ matrix.name }} Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }
# ]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# name: project.py
# on:
# schedule:
# - cron: '0 0 * * 5' # At 00:00 on Friday
# push:
# branches:
# - main
# jobs:
# run_Mealdb:
# runs-on: ubuntu-latest
# outputs:
# script_name: 'Mealdb Script'
# status: ${{ steps.set_status.outputs.status }}
# error: ${{ steps.set_status.outputs.error }}
# steps:
# - uses: actions/checkout@v2
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# restore-keys: |
# ${{ runner.os }}-pip-
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# pip install -r requirements.txt
# - name: Execute mealdb Scrapy spider
# id: mealdb_execution
# run: |
# scrapy crawl mealdb -O ./data/raw/mealdb.json . 2>mealdb_error.log || echo "error" > mealdb_error.log
# working-directory: ./landing_zone/collectors/Mealdb/mealscraper
# - name: Set job status and error
# id: set_status
# if: always()
# run: |
# if [ -s mealdb_error.log ]; then
# ERROR_MSG=$(<mealdb_error.log)
# echo "::set-output name=status::failure"
# echo "::set-output name=error::$ERROR_MSG"
# else
# echo "::set-output name=status::success"
# echo "::set-output name=error::None"
# fi
# run_ApprovedFood:
# runs-on: ubuntu-latest
# outputs:
# script_name: 'ApprovedFood Script'
# status: ${{ steps.set_status.outputs.status }}
# error: ${{ steps.set_status.outputs.error }}
# steps:
# - uses: actions/checkout@v2
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# pip install -r requirements.txt
# - name: Execute approvedfood Scrapy spider
# run: |
# scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json 2>approvedfood_error.log || echo "error" > approvedfood_error.log
# working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper
# - name: Set job status and error
# id: set_status
# if: always()
# run: |
# if [ -s approvedfood_error.log ]; then
# ERROR_MSG=$(<approvedfood_error.log)
# echo "::set-output name=status::failure"
# echo "::set-output name=error::$ERROR_MSG"
# else
# echo "::set-output name=status::success"
# echo "::set-output name=error::None"
# fi
# run_eat_by_date:
# runs-on: ubuntu-latest
# outputs:
# script_name: 'Eat by Date Script'
# status: ${{ steps.set_status.outputs.status }}
# error: ${{ steps.set_status.outputs.error }}
# steps:
# - uses: actions/checkout@v2
# - uses: actions/setup-python@v4
# with:
# python-version: '3.10'
# - uses: actions/cache@v3
# with:
# path: ~/.cache/pip
# key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
# - name: Install dependencies
# run: |
# python -m pip install --upgrade pip
# pip install -r requirements.txt
# - name: Execute eat_by_date.py script
# run: |
# python eat_by_date.py 2>eat_by_date_error.log || echo "error" > eat_by_date_error.log
# working-directory: ./landing_zone/collectors/eat_by_date
# - name: Set job status and error
# id: set_status
# if: always()
# run: |
# if [ -s eat_by_date_error.log ]; then
# ERROR_MSG=$(<eat_by_date_error.log)
# echo "::set-output name=status::failure"
# echo "::set-output name=error::$ERROR_MSG"
# else
# echo "::set-output name=status::success"
# echo "::set-output name=error::None"
# fi
# notify_slack:
# needs: [run_Mealdb, run_ApprovedFood] #, run_eat_by_date]
# runs-on: ubuntu-latest
# steps:
# - name: Set current date
# run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV
# - name: Notify Slack on Error
# uses: 8398a7/action-slack@v3
# with:
# status: custom
# custom_payload: |
# {
# "attachments": [{
# "color": "${{ needs.run_Mealdb.outputs.status }} === 'success' && '${{ needs.run_ApprovedFood.outputs.status }} === 'success' ? 'good' : 'danger'",
# "text": "Workflow: ${{ github.workflow }}\nDate: ${{ env.CURRENT_DATE }}\nJob: Detailed Status Update\n- Mealdb Script Status: ${{ needs.run_Mealdb.outputs.status }} (Error: ${{ needs.run_Mealdb.outputs.error }})\n- ApprovedFood Script Status: ${{ needs.run_ApprovedFood.outputs.status }} (Error: ${{ needs.run_ApprovedFood.outputs.error }})\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
# }]
# }
# env:
# SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
# CURRENT_DATE: ${{ env.CURRENT_DATE }}
# if: always()