project.yml

name: project.py

on:
  schedule:
    - cron: '0 0 * * 5' # At 00:00 on Friday
  push:
    branches:
      - main

jobs:
  run_Mealdb:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Cache Python packages
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies
        run: python -m pip install --upgrade pip && pip install -r requirements.txt

      - name: Execute mealdb Scrapy spider
        run: scrapy crawl mealdb -O ./data/raw/mealdb.json || echo "Script failed, check logs." > mealdb_error.log
        working-directory: ./landing_zone/collectors/meal_db/mealscraper
        # continue-on-error: false


      - name: Upload error log to Gist (mealdb)
        if: failure()
        id: upload_log
        run: |
          LOG_CONTENT=$(cat mealdb_error.log)
          JSON_PAYLOAD="{\"description\": \"Error Log for mealdb\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
          GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
          echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

      - name: Determine success/failure indicator and message (Mealdb)
        id: status_indicator
        run: |
          if [ -s mealdb_error.log ]; then
            echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
            echo "COLOR=#ff0000" >> $GITHUB_ENV
          else
            echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
            echo "COLOR=#36a64f" >> $GITHUB_ENV
          fi
        working-directory: ./landing_zone/collectors/meal_db/mealscraper

      - name: Determine Color
        id: set_color
        run: |
          if [ "${{ job.status }}" == "success" ]; then
            echo "::set-output name=color::#36a64f"
          else
            echo "::set-output name=color::#ff0000"
          fi

      - name: Set current date
        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

      - name: Notify Slack (Mealdb Script)
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              "attachments": [
                {
                   "color": "${{ steps.set_color.outputs.color }}",
                  "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: <$GIST_URL|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  run_ApprovedFood:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2



      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Cache Python packages
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies
        run: python -m pip install --upgrade pip && pip install -r requirements.txt

      - name: Execute approvedfood Scrapy spider
        run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json || echo "Script failed, check logs." > approvedfood_error.log
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
        # continue-on-error: false

      - name: Prepare error log (ApprovedFood)
        if: failure()
        run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper

      - name: Determine success/failure indicator and message (Mealdb)
        id: status_indicator
        run: |
          if [ -s approvedfood_error.log ]; then
            echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
            echo "COLOR=#ff0000" >> $GITHUB_ENV
          else
            echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
            echo "COLOR=#36a64f" >> $GITHUB_ENV
          fi
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper

      - name: Determine Color
        id: set_color
        run: |
          if [ "${{ job.status }}" == "success" ]; then
            echo "::set-output name=color::#36a64f"
          else
            echo "::set-output name=color::#ff0000"
          fi

      - name: Upload error log to Gist (appprovedfood)
        if: failure()
        id: upload_log
        run: |
          LOG_CONTENT=$(cat approvedfood_error.log)
          JSON_PAYLOAD="{\"description\": \"Error Log for Approvedfood\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
          GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
          echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

      - name: Set current date (ApprovedFood)
        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

      - name: Notify Slack (ApprovedFood Script)
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              "attachments": [
                {
                   "color": "${{ steps.set_color.outputs.color }}",
                  "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View URL>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  # run_eat_by_date:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Setup Python
  #       uses: actions/setup-python@v4
  #       with:
  #         python-version: '3.10'

  #     - name: Cache Python packages
  #       uses: actions/cache@v3
  #       with:
  #         path: ~/.cache/pip
  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #         restore-keys: |
  #           ${{ runner.os }}-pip-

  #     - name: Install dependencies
  #       run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #     - name: Execute eat_by_date.py script
  #       run: python eat_by_date.py || echo "Script failed, check logs." > eat_by_date_error.log
  #       working-directory: ./landing_zone/collectors/eat_by_date
  #       continue-on-error: true

  #     - name: Prepare error log (Eat by Date)
  #       if: failure()
  #       run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #       working-directory: ./landing_zone/collectors/eat_by_date

        # - name: Upload error log to Gist (Eat by Date)
        #   if: failure()
        #   id: upload_log
        #   run: |
        #     LOG_CONTENT=$(cat eat_by_date.log)
        #     JSON_PAYLOAD="{\"description\": \"Error Log for Eat by date\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
        #     GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
        #     echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #     - name: Set current date (Eat by Date)
  #       run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        # - name: Determine Color
        # id: set_color
        # run: |
        #   if [ "${{ job.status }}" == "success" ]; then
        #     echo "::set-output name=color::#36a64f"
        #   else
        #     echo "::set-output name=color::#ff0000"
        #   fi

  #     - name: Notify Slack (Eat by Date Script)
  #       if: always()
  #       uses: 8398a7/action-slack@v3
  #       with:
  #         status: custom
  #         custom_payload: |
  #           {
  #             "attachments": [
  #               {
  #                 "color": "${{ steps.set_color.outputs.color }}",
  #                 "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #               }
  #             ]
  #           }
  #       env:
  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

# run_flipkart:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v2

#       - name: Setup Python
#         uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - name: Cache Python packages
#         uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: |
#             ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: python -m pip install --upgrade pip && pip install -r requirements.txt

#       - name: Execute scrap_flipkart_pages script
#         run: python scrap_flipkart_pages_sel.py || echo "Script failed, check logs." > flipkart_error.log
#         working-directory: ./landing_zone/collectors/Flipkart
#         continue-on-error: true

#       - name: Prepare error log (flipkart)
#         if: failure()
#         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && flipkart_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
#         working-directory: ./landing_zone/collectors/Flipkart

#       - name: Upload error log to Gist (flipkart)
#         if: failure()
#         id: upload_log
#         run: |
#           LOG_CONTENT=$(cat flipkart_error.log)
#           JSON_PAYLOAD="{\"description\": \"Error Log for flipkat\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
#           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
#           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

#       - name: Determine success/failure indicator and message (Mealdb)
#         id: status_indicator
#         run: |
#           if [ -s flipkart_error.log ]; then
#             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
#             echo "COLOR=#ff0000" >> $GITHUB_ENV
#           else
#             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
#             echo "COLOR=#36a64f" >> $GITHUB_ENV
#           fi
#         working-directory: ./landing_zone/collectors/Flipkart

        # - name: Determine Color
        #   id: set_color
        #   run: |
        #     if [ "${{ job.status }}" == "success" ]; then
        #       echo "::set-output name=color::#36a64f"
        #     else
        #       echo "::set-output name=color::#ff0000"
        #     fi


#       - name: Set current date (flipkart)
#         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

#       - name: Notify Slack (flipkart Script)
#         if: always()
#         uses: 8398a7/action-slack@v3
#         with:
#           status: custom
#           custom_payload: |
#             {
#               "attachments": [
#                 {
#                   "color": "${{ steps.set_color.outputs.color }}",
#                   "text": "Flipkart Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
#                 }
#               ]
#             }
#         env:
#           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"


  # run_big_basket:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute bigBasket script
  #         run: python big_basket.py || echo "Script failed, check logs." > bigBasket.log
  #         working-directory: ./landing_zone/collectors/big_basket
  #         # continue-on-error: false

  #       - name: Prepare error log (bigBasket)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat bigBasket.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/collectors/big_basket

  #       - name: Upload error log to Gist (bigbasket)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat bigBasket.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for Bigbasket\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (bigBasket)
  #         id: status_indicator
  #         run: |
  #           if [ -s bigBasket.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/collectors/big_basket

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi

  #       - name: Set current date (bigBasket)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (bigBasket Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "BigBasket Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  # run_catalonia_establishment_locationt:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute catalonia_establishment_location script
  #         run: python establishments_scraper.py || echo "Script failed, check logs." > catalonia_establishment_location.log
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location
  #         continue-on-error: true

  #       - name: Prepare error log (establishment location)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat catalonia_establishment_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location

  #       - name: Upload error log to Gist (Establishment Location)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat catalonia_establishment_location.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for Establishment Location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (catalonia_establishment_location)
  #         id: status_indicator
  #         run: |
  #           if [ -s catalonia_establishment_location.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (catalonia_establishment_location)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (catalonia_establishment_location Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Catalonia Establishment Location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"




  # run_OCR:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute OCR script
  #         run: python ocr_marcadona.py || echo "Script failed, check logs." > ocr.log
  #         working-directory: ./landing_zone/collectors/OCR
  #         # continue-on-error: false

  #       - name: Prepare error log (OCR)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat ocr.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/collectors/OCR

  #       - name: Upload error log to Gist (ocr)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat ocr.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for ocr\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (OCR)
  #         id: status_indicator
  #         run: |
  #           if [ -s OCR.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/collectors/OCR

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi

  #       - name: Set current date (OCR)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (OCR Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "OCR Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  # run_customers:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute customer script
  #         run: python sm_retail_customer.py || echo "Script failed, check logs." > customer.log
  #         working-directory: ./landing_zone/collectors/customers
  #         # continue-on-error: false

  #       - name: Prepare error log (customers)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customers.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/collectors/customers

  #       - name: Upload error log to Gist (customers)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat customers.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for customers\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (customers)
  #         id: status_indicator
  #         run: |
  #           if [ -s customers.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/collectors/customers

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (customers)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (customers Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Customers Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  # run_customer_location:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute customer_location script
  #         run: python customer_location.py || echo "Script failed, check logs." > customer_location.log
  #         working-directory: ./landing_zone/synthetic/customer_location
  #         # continue-on-error: false

  #       - name: Prepare error log (customer_location)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/synthetic/customer_location

  #       - name: Upload error log to Gist (customer location)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat customer_location.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for customer location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (customer_location)
  #         id: status_indicator
  #         run: |
  #           if [ -s customer_location.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/synthetic/customer_location

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (customer_location)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (customer_location Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Customer location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"




  # run_customer_purchase:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute customer_purchase script
  #         run: python customer_purchase.py || echo "Script failed, check logs." > customer_purchase.log
  #         working-directory: ./landing_zone/synthetic/customer_purchase
  #         # continue-on-error: false

  #       - name: Prepare error log (customer_purchase)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_purchase.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/synthetic/customer_purchase

  #       - name: Upload error log to Gist (customer purchase)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat customer_purchase.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for customer purchase\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (customer_purchase)
  #         id: status_indicator
  #         run: |
  #           if [ -s customer_purchase.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/synthetic/customer_purchase

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (customer_purchase)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (customer_purchase Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Customer purchase Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  # run_sentiment_reviews:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute sentiment reviews script
  #         run: python business_reviews.py  || echo "Script failed, check logs." > sentiment_reviews.log
  #         working-directory: ./landing_zone/synthetic/sentiment reviews
  #         # continue-on-error: false

  #       - name: Execute sentiment reviews script
  #         run: python individual_reviews.py  || echo "Script failed, check logs." > sentiment_reviews.log
  #         working-directory: ./landing_zone/synthetic/sentiment reviews
  #         # continue-on-error: false

  #       - name: Prepare error log (sentiment reviews)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat sentiment_reviews.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/synthetic/sentiment reviews

  #       - name: Upload error log to Gist (sentiment_reviews)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat sentiment_reviews.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for sentiment reviews\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (sentiment reviews)
  #         id: status_indicator
  #         run: |
  #           if [ -s sentiment_reviews.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/synthetic/sentiment reviews

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (sentiment reviews)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (sentiment reviews Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Sentiment reviews Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  # run_supermarket_products:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute supermarket_products script
  #         run: python supermarket_products.py || echo "Script failed, check logs." > supermarket_products.log
  #         working-directory: ./landing_zone/synthetic/supermarket_products
  #         # continue-on-error: false

  #       - name: Prepare error log (supermarket_products)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat supermarket_products.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/synthetic/supermarket_products

  #       - name: Upload error log to Gist (supermarket_products)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat supermarket_products.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for supermarket products\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (supermarket_products)
  #         id: status_indicator
  #         run: |
  #           if [ -s supermarket_products.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/synthetic/supermarket_products

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi



  #       - name: Set current date (supermarket_products)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (supermarket_products Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Supermarket products Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"