initial code for dag synthetic and collector #107
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # name: project.py | |
| # on: | |
| # schedule: | |
| # - cron: '0 0 * * 5' # At 00:00 on Friday | |
| # push: | |
| # branches: | |
| # - main | |
| # jobs: | |
| # run_Mealdb: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute mealdb Scrapy spider | |
| # run: scrapy crawl mealdb -O ./data/raw/mealdb.json | |
| # working-directory: ./landing_zone/collectors/meal_db/mealscraper | |
| # continue-on-error: true | |
| # - name: Prepare error log | |
| # id: prep_log | |
| # if: failure() | |
| # run: | | |
| # echo "ERROR_LOG<<EOF" >> $GITHUB_ENV | |
| # cat mealdb_error.log >> $GITHUB_ENV | |
| # echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/meal_db/mealscraper | |
| # - name: Determine success/failure indicator and message (Mealdb) | |
| # id: status_indicator | |
| # run: | | |
| # if [ -s mealdb_error.log ]; then | |
| # echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| # echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| # else | |
| # echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| # echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| # fi | |
| # working-directory: ./landing_zone/collectors/meal_db/mealscraper | |
| # - name: Set current date | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack (Mealdb Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ env.COLOR }}", | |
| # "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # run_ApprovedFood: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute approvedfood Scrapy spider | |
| # run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json | |
| # working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| # continue-on-error: true | |
| # - name: Prepare error log (ApprovedFood) | |
| # if: failure() | |
| # run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| # - name: Determine success/failure indicator and message (Mealdb) | |
| # id: status_indicator | |
| # run: | | |
| # if [ -s mealdb_error.log ]; then | |
| # echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| # echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| # else | |
| # echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| # echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| # fi | |
| # working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| # - name: Set current date (ApprovedFood) | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack (ApprovedFood Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ env.COLOR }}", | |
| # "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # run_eat_by_date: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute eat_by_date.py script | |
| # run: python eat_by_date.py | |
| # working-directory: ./landing_zone/collectors/eat_by_date | |
| # continue-on-error: true | |
| # - name: Prepare error log (Eat by Date) | |
| # if: failure() | |
| # run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/eat_by_date | |
| # - name: Set current date (Eat by Date) | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack (Eat by Date Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ job.status }} == 'success' ? 'good' : 'danger'", | |
| # "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| name: project.py | |
| on: | |
| schedule: | |
| - cron: '0 0 * * 5' # At 00:00 on Friday | |
| push: | |
| branches: | |
| - main | |
| jobs: | |
| run_Mealdb: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute mealdb Scrapy spider | |
| run: scrapy crawl mealdb -O ./data/raw/mealdb.json || echo "Script failed, check logs." > mealdb_error.log | |
| working-directory: ./landing_zone/collectors/meal_db/mealscraper | |
| # continue-on-error: false | |
| - name: Upload error log to Gist (mealdb) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat mealdb_error.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for mealdb\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (Mealdb) | |
| id: status_indicator | |
| run: | | |
| if [ -s mealdb_error.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/collectors/meal_db/mealscraper | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (Mealdb Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: <$GIST_URL|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_ApprovedFood: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute approvedfood Scrapy spider | |
| run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json || echo "Script failed, check logs." > approvedfood_error.log | |
| working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| # continue-on-error: false | |
| - name: Prepare error log (ApprovedFood) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| - name: Determine success/failure indicator and message (Mealdb) | |
| id: status_indicator | |
| run: | | |
| if [ -s approvedfood_error.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Upload error log to Gist (appprovedfood) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat approvedfood_error.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for Approvedfood\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Set current date (ApprovedFood) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (ApprovedFood Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View URL>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # run_eat_by_date: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute eat_by_date.py script | |
| # run: python eat_by_date.py || echo "Script failed, check logs." > eat_by_date_error.log | |
| # working-directory: ./landing_zone/collectors/eat_by_date | |
| # continue-on-error: true | |
| # - name: Prepare error log (Eat by Date) | |
| # if: failure() | |
| # run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/eat_by_date | |
| # - name: Upload error log to Gist (Eat by Date) | |
| # if: failure() | |
| # id: upload_log | |
| # run: | | |
| # LOG_CONTENT=$(cat eat_by_date.log) | |
| # JSON_PAYLOAD="{\"description\": \"Error Log for Eat by date\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| # GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| # echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| # - name: Set current date (Eat by Date) | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Determine Color | |
| # id: set_color | |
| # run: | | |
| # if [ "${{ job.status }}" == "success" ]; then | |
| # echo "::set-output name=color::#36a64f" | |
| # else | |
| # echo "::set-output name=color::#ff0000" | |
| # fi | |
| # - name: Notify Slack (Eat by Date Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ steps.set_color.outputs.color }}", | |
| # "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} | |
| # run_flipkart: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute scrap_flipkart_pages script | |
| # run: python scrap_flipkart_pages_sel.py || echo "Script failed, check logs." > flipkart_error.log | |
| # working-directory: ./landing_zone/collectors/Flipkart | |
| # continue-on-error: true | |
| # - name: Prepare error log (flipkart) | |
| # if: failure() | |
| # run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && flipkart_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/Flipkart | |
| # - name: Upload error log to Gist (flipkart) | |
| # if: failure() | |
| # id: upload_log | |
| # run: | | |
| # LOG_CONTENT=$(cat flipkart_error.log) | |
| # JSON_PAYLOAD="{\"description\": \"Error Log for flipkat\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| # GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| # echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| # - name: Determine success/failure indicator and message (Mealdb) | |
| # id: status_indicator | |
| # run: | | |
| # if [ -s flipkart_error.log ]; then | |
| # echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| # echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| # else | |
| # echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| # echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| # fi | |
| # working-directory: ./landing_zone/collectors/Flipkart | |
| # - name: Determine Color | |
| # id: set_color | |
| # run: | | |
| # if [ "${{ job.status }}" == "success" ]; then | |
| # echo "::set-output name=color::#36a64f" | |
| # else | |
| # echo "::set-output name=color::#ff0000" | |
| # fi | |
| # - name: Set current date (flipkart) | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack (flipkart Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ steps.set_color.outputs.color }}", | |
| # "text": "Flipkart Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_big_basket: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute bigBasket script | |
| run: python big_basket.py || echo "Script failed, check logs." > bigBasket.log | |
| working-directory: ./landing_zone/collectors/big_basket | |
| # continue-on-error: false | |
| - name: Prepare error log (bigBasket) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat bigBasket.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/collectors/big_basket | |
| - name: Upload error log to Gist (bigbasket) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat bigBasket.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for Bigbasket\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (bigBasket) | |
| id: status_indicator | |
| run: | | |
| if [ -s bigBasket.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/collectors/big_basket | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (bigBasket) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (bigBasket Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "BigBasket Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # run_catalonia_establishment_locationt: | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute catalonia_establishment_location script | |
| # run: python establishments_scraper.py || echo "Script failed, check logs." > catalonia_establishment_location.log | |
| # working-directory: ./landing_zone/collectors/catalonia_establishment_location | |
| # continue-on-error: true | |
| # - name: Prepare error log (establishment location) | |
| # if: failure() | |
| # run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat catalonia_establishment_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| # working-directory: ./landing_zone/collectors/catalonia_establishment_location | |
| # - name: Upload error log to Gist (Establishment Location) | |
| # if: failure() | |
| # id: upload_log | |
| # run: | | |
| # LOG_CONTENT=$(cat catalonia_establishment_location.log) | |
| # JSON_PAYLOAD="{\"description\": \"Error Log for Establishment Location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| # GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| # echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| # - name: Determine success/failure indicator and message (catalonia_establishment_location) | |
| # id: status_indicator | |
| # run: | | |
| # if [ -s catalonia_establishment_location.log ]; then | |
| # echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| # echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| # else | |
| # echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| # echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| # fi | |
| # working-directory: ./landing_zone/collectors/catalonia_establishment_location | |
| # - name: Determine Color | |
| # id: set_color | |
| # run: | | |
| # if [ "${{ job.status }}" == "success" ]; then | |
| # echo "::set-output name=color::#36a64f" | |
| # else | |
| # echo "::set-output name=color::#ff0000" | |
| # fi | |
| # - name: Set current date (catalonia_establishment_location) | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack (catalonia_establishment_location Script) | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ steps.set_color.outputs.color }}", | |
| # "text": "Catalonia Establishment Location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_OCR: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute OCR script | |
| run: python ocr_marcadona.py || echo "Script failed, check logs." > ocr.log | |
| working-directory: ./landing_zone/collectors/OCR | |
| # continue-on-error: false | |
| - name: Prepare error log (OCR) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat ocr.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/collectors/OCR | |
| - name: Upload error log to Gist (ocr) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat ocr.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for ocr\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (OCR) | |
| id: status_indicator | |
| run: | | |
| if [ -s OCR.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/collectors/OCR | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (OCR) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (OCR Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "OCR Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_customers: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute customer script | |
| run: python sm_retail_customer.py || echo "Script failed, check logs." > customer.log | |
| working-directory: ./landing_zone/collectors/customers | |
| # continue-on-error: false | |
| - name: Prepare error log (customers) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customers.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/collectors/customers | |
| - name: Upload error log to Gist (customers) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat customers.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for customers\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (customers) | |
| id: status_indicator | |
| run: | | |
| if [ -s customers.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/collectors/customers | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (customers) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (customers Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Customers Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_customer_location: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute customer_location script | |
| run: python customer_location.py || echo "Script failed, check logs." > customer_location.log | |
| working-directory: ./landing_zone/synthetic/customer_location | |
| # continue-on-error: false | |
| - name: Prepare error log (customer_location) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/synthetic/customer_location | |
| - name: Upload error log to Gist (customer location) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat customer_location.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for customer location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (customer_location) | |
| id: status_indicator | |
| run: | | |
| if [ -s customer_location.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/synthetic/customer_location | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (customer_location) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (customer_location Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Customer location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_customer_purchase: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute customer_purchase script | |
| run: python customer_purchase.py || echo "Script failed, check logs." > customer_purchase.log | |
| working-directory: ./landing_zone/synthetic/customer_purchase | |
| # continue-on-error: false | |
| - name: Prepare error log (customer_purchase) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_purchase.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/synthetic/customer_purchase | |
| - name: Upload error log to Gist (customer purchase) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat customer_purchase.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for customer purchase\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (customer_purchase) | |
| id: status_indicator | |
| run: | | |
| if [ -s customer_purchase.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/synthetic/customer_purchase | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (customer_purchase) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (customer_purchase Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Customer purchase Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_sentiment_reviews: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute sentiment reviews script | |
| run: python business_reviews.py || echo "Script failed, check logs." > sentiment_reviews.log | |
| working-directory: ./landing_zone/synthetic/sentiment reviews | |
| # continue-on-error: false | |
| - name: Execute sentiment reviews script | |
| run: python individual_reviews.py || echo "Script failed, check logs." > sentiment_reviews.log | |
| working-directory: ./landing_zone/synthetic/sentiment reviews | |
| # continue-on-error: false | |
| - name: Prepare error log (sentiment reviews) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat sentiment_reviews.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/synthetic/sentiment reviews | |
| - name: Upload error log to Gist (sentiment_reviews) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat sentiment_reviews.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for sentiment reviews\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (sentiment reviews) | |
| id: status_indicator | |
| run: | | |
| if [ -s sentiment_reviews.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/synthetic/sentiment reviews | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (sentiment reviews) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (sentiment reviews Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Sentiment reviews Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| run_supermarket_products: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache Python packages | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pip- | |
| - name: Install dependencies | |
| run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| - name: Execute supermarket_products script | |
| run: python supermarket_products.py || echo "Script failed, check logs." > supermarket_products.log | |
| working-directory: ./landing_zone/synthetic/supermarket_products | |
| # continue-on-error: false | |
| - name: Prepare error log (supermarket_products) | |
| if: failure() | |
| run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat supermarket_products.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV | |
| working-directory: ./landing_zone/synthetic/supermarket_products | |
| - name: Upload error log to Gist (supermarket_products) | |
| if: failure() | |
| id: upload_log | |
| run: | | |
| LOG_CONTENT=$(cat supermarket_products.log) | |
| JSON_PAYLOAD="{\"description\": \"Error Log for supermarket products\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| - name: Determine success/failure indicator and message (supermarket_products) | |
| id: status_indicator | |
| run: | | |
| if [ -s supermarket_products.log ]; then | |
| echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV | |
| echo "COLOR=#ff0000" >> $GITHUB_ENV | |
| else | |
| echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV | |
| echo "COLOR=#36a64f" >> $GITHUB_ENV | |
| fi | |
| working-directory: ./landing_zone/synthetic/supermarket_products | |
| - name: Determine Color | |
| id: set_color | |
| run: | | |
| if [ "${{ job.status }}" == "success" ]; then | |
| echo "::set-output name=color::#36a64f" | |
| else | |
| echo "::set-output name=color::#ff0000" | |
| fi | |
| - name: Set current date (supermarket_products) | |
| run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| - name: Notify Slack (supermarket_products Script) | |
| if: always() | |
| uses: 8398a7/action-slack@v3 | |
| with: | |
| status: custom | |
| custom_payload: | | |
| { | |
| "attachments": [ | |
| { | |
| "color": "${{ steps.set_color.outputs.color }}", | |
| "text": "Supermarket products Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| } | |
| ] | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # name: project.py | |
| # on: | |
| # schedule: | |
| # - cron: '0 0 * * 5' # At 00:00 on Friday | |
| # push: | |
| # branches: | |
| # - main | |
| # jobs: | |
| # run_scripts: | |
| # runs-on: ubuntu-latest | |
| # strategy: | |
| # matrix: | |
| # include: | |
| # - name: 'ocr' | |
| # path: 'OCR' | |
| # command: 'python ocr_marcadona.py' | |
| # - name: 'customers' | |
| # path: 'customers' | |
| # command: 'python sm_retail_customer.py' | |
| # # - name: 'catalonia_establishment_location' | |
| # # path: 'catalonia_establishment_location' | |
| # # command: 'python establishments_scraper.py' | |
| # # - name: 'flipkart' | |
| # # path: 'Flipkart' | |
| # # command: 'python scrap_flipkart_pages_sel.py' | |
| # # - name: 'Eatbydate' | |
| # # path: 'eat_by_date' | |
| # # command: 'python eat_by_date.py' | |
| # - name: 'Mealdb' | |
| # path: 'meal_db/mealscraper' | |
| # command: 'scrapy crawl mealdb -O ./data/raw/mealdb.json' | |
| # - name: 'ApprovedFood' | |
| # path: 'approved_food_uk/approvedfood_scraper' | |
| # command: 'scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json' | |
| # - name: 'BigBasket' | |
| # path: 'big_basket' | |
| # command: 'python big_basket.py' | |
| # - name: 'CustomerLocation' | |
| # path: 'synthetic/customer_location' | |
| # command: 'python customer_location.py' | |
| # - name: 'CustomerPurchase' | |
| # path: 'synthetic/customer_purchase' | |
| # command: 'python customer_purchase.py' | |
| # - name: 'Business_SentimentReview' | |
| # path: 'synthetic/sentiment reviews' | |
| # command: 'python business_reviews.py' | |
| # - name: 'Individual_SentimentReview' | |
| # path: 'synthetic/sentiment reviews' | |
| # command: 'python individual_reviews.py' | |
| # - name: 'SupermarketProducts' | |
| # path: 'synthetic/supermarket_products' | |
| # command: 'python supermarket_products.py' | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - name: Setup Python | |
| # uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - name: Cache Python packages | |
| # uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: python -m pip install --upgrade pip && pip install -r requirements.txt | |
| # - name: Execute Script | |
| # run: | | |
| # cd ./landing_zone/collectors/${{ matrix.path }} | |
| # ${{ matrix.command }} | |
| # continue-on-error: true | |
| # - name: Determine Color | |
| # id: set color | |
| # run: | | |
| # if [ "${{ job.status }}" == "success" ]; then | |
| # echo "::set-output name=color::#36a64f" | |
| # else | |
| # echo "::set-output name=color::#ff0000" | |
| # fi | |
| # - name: Prepare error log | |
| # if: failure() | |
| # run: | | |
| # echo "ERROR_LOG<<EOF" >> $GITHUB_ENV | |
| # cat ./logs/${{ matrix.name }}_error.log >> $GITHUB_ENV | |
| # echo "EOF" >> $GITHUB_ENV | |
| # - name: Upload error log to Gist | |
| # if: failure() | |
| # run: | | |
| # LOG_CONTENT=$(cat ./logs/${{ matrix.name }}_error.log) | |
| # JSON_PAYLOAD="{\"description\": \"Error Log for ${{ matrix.name }}\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}" | |
| # GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url') | |
| # echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV | |
| # - name: Notify Slack | |
| # if: always() | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [ | |
| # { | |
| # "color": "${{ steps.set_color.outputs.color }}", | |
| # "text": "${{ matrix.name }} Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # } | |
| # ] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # name: project.py | |
| # on: | |
| # schedule: | |
| # - cron: '0 0 * * 5' # At 00:00 on Friday | |
| # push: | |
| # branches: | |
| # - main | |
| # jobs: | |
| # run_Mealdb: | |
| # runs-on: ubuntu-latest | |
| # outputs: | |
| # script_name: 'Mealdb Script' | |
| # status: ${{ steps.set_status.outputs.status }} | |
| # error: ${{ steps.set_status.outputs.error }} | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # restore-keys: | | |
| # ${{ runner.os }}-pip- | |
| # - name: Install dependencies | |
| # run: | | |
| # python -m pip install --upgrade pip | |
| # pip install -r requirements.txt | |
| # - name: Execute mealdb Scrapy spider | |
| # id: mealdb_execution | |
| # run: | | |
| # scrapy crawl mealdb -O ./data/raw/mealdb.json . 2>mealdb_error.log || echo "error" > mealdb_error.log | |
| # working-directory: ./landing_zone/collectors/Mealdb/mealscraper | |
| # - name: Set job status and error | |
| # id: set_status | |
| # if: always() | |
| # run: | | |
| # if [ -s mealdb_error.log ]; then | |
| # ERROR_MSG=$(<mealdb_error.log) | |
| # echo "::set-output name=status::failure" | |
| # echo "::set-output name=error::$ERROR_MSG" | |
| # else | |
| # echo "::set-output name=status::success" | |
| # echo "::set-output name=error::None" | |
| # fi | |
| # run_ApprovedFood: | |
| # runs-on: ubuntu-latest | |
| # outputs: | |
| # script_name: 'ApprovedFood Script' | |
| # status: ${{ steps.set_status.outputs.status }} | |
| # error: ${{ steps.set_status.outputs.error }} | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # - name: Install dependencies | |
| # run: | | |
| # python -m pip install --upgrade pip | |
| # pip install -r requirements.txt | |
| # - name: Execute approvedfood Scrapy spider | |
| # run: | | |
| # scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json 2>approvedfood_error.log || echo "error" > approvedfood_error.log | |
| # working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper | |
| # - name: Set job status and error | |
| # id: set_status | |
| # if: always() | |
| # run: | | |
| # if [ -s approvedfood_error.log ]; then | |
| # ERROR_MSG=$(<approvedfood_error.log) | |
| # echo "::set-output name=status::failure" | |
| # echo "::set-output name=error::$ERROR_MSG" | |
| # else | |
| # echo "::set-output name=status::success" | |
| # echo "::set-output name=error::None" | |
| # fi | |
| # run_eat_by_date: | |
| # runs-on: ubuntu-latest | |
| # outputs: | |
| # script_name: 'Eat by Date Script' | |
| # status: ${{ steps.set_status.outputs.status }} | |
| # error: ${{ steps.set_status.outputs.error }} | |
| # steps: | |
| # - uses: actions/checkout@v2 | |
| # - uses: actions/setup-python@v4 | |
| # with: | |
| # python-version: '3.10' | |
| # - uses: actions/cache@v3 | |
| # with: | |
| # path: ~/.cache/pip | |
| # key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} | |
| # - name: Install dependencies | |
| # run: | | |
| # python -m pip install --upgrade pip | |
| # pip install -r requirements.txt | |
| # - name: Execute eat_by_date.py script | |
| # run: | | |
| # python eat_by_date.py 2>eat_by_date_error.log || echo "error" > eat_by_date_error.log | |
| # working-directory: ./landing_zone/collectors/eat_by_date | |
| # - name: Set job status and error | |
| # id: set_status | |
| # if: always() | |
| # run: | | |
| # if [ -s eat_by_date_error.log ]; then | |
| # ERROR_MSG=$(<eat_by_date_error.log) | |
| # echo "::set-output name=status::failure" | |
| # echo "::set-output name=error::$ERROR_MSG" | |
| # else | |
| # echo "::set-output name=status::success" | |
| # echo "::set-output name=error::None" | |
| # fi | |
| # notify_slack: | |
| # needs: [run_Mealdb, run_ApprovedFood] #, run_eat_by_date] | |
| # runs-on: ubuntu-latest | |
| # steps: | |
| # - name: Set current date | |
| # run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV | |
| # - name: Notify Slack on Error | |
| # uses: 8398a7/action-slack@v3 | |
| # with: | |
| # status: custom | |
| # custom_payload: | | |
| # { | |
| # "attachments": [{ | |
| # "color": "${{ needs.run_Mealdb.outputs.status }} === 'success' && '${{ needs.run_ApprovedFood.outputs.status }} === 'success' ? 'good' : 'danger'", | |
| # "text": "Workflow: ${{ github.workflow }}\nDate: ${{ env.CURRENT_DATE }}\nJob: Detailed Status Update\n- Mealdb Script Status: ${{ needs.run_Mealdb.outputs.status }} (Error: ${{ needs.run_Mealdb.outputs.error }})\n- ApprovedFood Script Status: ${{ needs.run_ApprovedFood.outputs.status }} (Error: ${{ needs.run_ApprovedFood.outputs.error }})\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}" | |
| # }] | |
| # } | |
| # env: | |
| # SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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" | |
| # CURRENT_DATE: ${{ env.CURRENT_DATE }} | |
| # if: always() | |