project.yml

# name: project.py

# on:
#   schedule:
#     - cron: '0 0 * * 5' # At 00:00 on Friday
#   push:
#     branches:
#       - main

# jobs:
#   run_Mealdb:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v2
      
#       - name: Setup Python
#         uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - name: Cache Python packages
#         uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: |
#             ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: python -m pip install --upgrade pip && pip install -r requirements.txt

#       - name: Execute mealdb Scrapy spider
#         run: scrapy crawl mealdb -O ./data/raw/mealdb.json
#         working-directory: ./landing_zone/collectors/meal_db/mealscraper
#         continue-on-error: true

#       - name: Prepare error log
#         id: prep_log
#         if: failure()
#         run: |
#           echo "ERROR_LOG<<EOF" >> $GITHUB_ENV
#           cat mealdb_error.log >> $GITHUB_ENV
#           echo "EOF" >> $GITHUB_ENV
#         working-directory: ./landing_zone/collectors/meal_db/mealscraper

#       - name: Determine success/failure indicator and message (Mealdb)
#         id: status_indicator
#         run: |
#           if [ -s mealdb_error.log ]; then
#             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
#             echo "COLOR=#ff0000" >> $GITHUB_ENV
#           else
#             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
#             echo "COLOR=#36a64f" >> $GITHUB_ENV
#           fi
#         working-directory: ./landing_zone/collectors/meal_db/mealscraper


#       - name: Set current date
#         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

#       - name: Notify Slack (Mealdb Script)
#         if: always()
#         uses: 8398a7/action-slack@v3
#         with:
#           status: custom
#           custom_payload: |
#             {
#               "attachments": [
#                 {
#                    "color": "${{ env.COLOR }}",
#                   "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
#                 }
#               ]
#             }
#         env:
#           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

#   run_ApprovedFood:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v2

#       - name: Setup Python
#         uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - name: Cache Python packages
#         uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: |
#             ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: python -m pip install --upgrade pip && pip install -r requirements.txt

#       - name: Execute approvedfood Scrapy spider
#         run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json
#         working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
#         continue-on-error: true

#       - name: Prepare error log (ApprovedFood)
#         if: failure()
#         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
#         working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper

#       - name: Determine success/failure indicator and message (Mealdb)
#         id: status_indicator
#         run: |
#           if [ -s mealdb_error.log ]; then
#             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
#             echo "COLOR=#ff0000" >> $GITHUB_ENV
#           else
#             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
#             echo "COLOR=#36a64f" >> $GITHUB_ENV
#           fi
#         working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper


#       - name: Set current date (ApprovedFood)
#         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

#       - name: Notify Slack (ApprovedFood Script)
#         if: always()
#         uses: 8398a7/action-slack@v3
#         with:
#           status: custom
#           custom_payload: |
#             {
#               "attachments": [
#                 {
#                    "color": "${{ env.COLOR }}",
#                   "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
#                 }
#               ]
#             }
#         env:
#           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  # run_eat_by_date:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Setup Python
  #       uses: actions/setup-python@v4
  #       with:
  #         python-version: '3.10'

  #     - name: Cache Python packages
  #       uses: actions/cache@v3
  #       with:
  #         path: ~/.cache/pip
  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #         restore-keys: |
  #           ${{ runner.os }}-pip-

  #     - name: Install dependencies
  #       run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #     - name: Execute eat_by_date.py script
  #       run: python eat_by_date.py
  #       working-directory: ./landing_zone/collectors/eat_by_date
  #       continue-on-error: true

  #     - name: Prepare error log (Eat by Date)
  #       if: failure()
  #       run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #       working-directory: ./landing_zone/collectors/eat_by_date

  #     - name: Set current date (Eat by Date)
  #       run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #     - name: Notify Slack (Eat by Date Script)
  #       if: always()
  #       uses: 8398a7/action-slack@v3
  #       with:
  #         status: custom
  #         custom_payload: |
  #           {
  #             "attachments": [
  #               {
  #                 "color": "${{ job.status }} == 'success' ? 'good' : 'danger'",
  #                 "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: ${{ env.ERROR_LOG }}\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #               }
  #             ]
  #           }
  #       env:
  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}




name: project.py

on:
  schedule:
    - cron: '0 0 * * 5' # At 00:00 on Friday
  push:
    branches:
      - main

jobs:
  run_Mealdb:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Cache Python packages
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies
        run: python -m pip install --upgrade pip && pip install -r requirements.txt

      - name: Execute mealdb Scrapy spider
        run: scrapy crawl mealdb -O ./data/raw/mealdb.json || echo "Script failed, check logs." > mealdb_error.log
        working-directory: ./landing_zone/collectors/meal_db/mealscraper
        # continue-on-error: false


      - name: Upload error log to Gist (mealdb)
        if: failure()
        id: upload_log
        run: |
          LOG_CONTENT=$(cat mealdb_error.log)
          JSON_PAYLOAD="{\"description\": \"Error Log for mealdb\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
          GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
          echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

      - name: Determine success/failure indicator and message (Mealdb)
        id: status_indicator
        run: |
          if [ -s mealdb_error.log ]; then
            echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
            echo "COLOR=#ff0000" >> $GITHUB_ENV
          else
            echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
            echo "COLOR=#36a64f" >> $GITHUB_ENV
          fi
        working-directory: ./landing_zone/collectors/meal_db/mealscraper

      - name: Determine Color
        id: set_color
        run: |
          if [ "${{ job.status }}" == "success" ]; then
            echo "::set-output name=color::#36a64f"
          else
            echo "::set-output name=color::#ff0000"
          fi

      - name: Set current date
        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

      - name: Notify Slack (Mealdb Script)
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              "attachments": [
                {
                   "color": "${{ steps.set_color.outputs.color }}",
                  "text": "Mealdb Script\nStatus: ${{ job.status }}\nError Log: <$GIST_URL|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  run_ApprovedFood:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2



      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Cache Python packages
        uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ runner.os }}-pip-

      - name: Install dependencies
        run: python -m pip install --upgrade pip && pip install -r requirements.txt

      - name: Execute approvedfood Scrapy spider
        run: scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json || echo "Script failed, check logs." > approvedfood_error.log
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper
        # continue-on-error: false

      - name: Prepare error log (ApprovedFood)
        if: failure()
        run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat approvedfood_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper

      - name: Determine success/failure indicator and message (Mealdb)
        id: status_indicator
        run: |
          if [ -s approvedfood_error.log ]; then
            echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
            echo "COLOR=#ff0000" >> $GITHUB_ENV
          else
            echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
            echo "COLOR=#36a64f" >> $GITHUB_ENV
          fi
        working-directory: ./landing_zone/collectors/approved_food_uk/approvedfood_scraper

      - name: Determine Color
        id: set_color
        run: |
          if [ "${{ job.status }}" == "success" ]; then
            echo "::set-output name=color::#36a64f"
          else
            echo "::set-output name=color::#ff0000"
          fi

      - name: Upload error log to Gist (appprovedfood)
        if: failure()
        id: upload_log
        run: |
          LOG_CONTENT=$(cat approvedfood_error.log)
          JSON_PAYLOAD="{\"description\": \"Error Log for Approvedfood\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
          GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
          echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

      - name: Set current date (ApprovedFood)
        run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

      - name: Notify Slack (ApprovedFood Script)
        if: always()
        uses: 8398a7/action-slack@v3
        with:
          status: custom
          custom_payload: |
            {
              "attachments": [
                {
                   "color": "${{ steps.set_color.outputs.color }}",
                  "text": "ApprovedFood Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View URL>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                }
              ]
            }
        env:
          SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  # run_eat_by_date:
  #   runs-on: ubuntu-latest
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Setup Python
  #       uses: actions/setup-python@v4
  #       with:
  #         python-version: '3.10'

  #     - name: Cache Python packages
  #       uses: actions/cache@v3
  #       with:
  #         path: ~/.cache/pip
  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #         restore-keys: |
  #           ${{ runner.os }}-pip-

  #     - name: Install dependencies
  #       run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #     - name: Execute eat_by_date.py script
  #       run: python eat_by_date.py || echo "Script failed, check logs." > eat_by_date_error.log
  #       working-directory: ./landing_zone/collectors/eat_by_date
  #       continue-on-error: true

  #     - name: Prepare error log (Eat by Date)
  #       if: failure()
  #       run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat eat_by_date_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #       working-directory: ./landing_zone/collectors/eat_by_date

        # - name: Upload error log to Gist (Eat by Date)
        #   if: failure()
        #   id: upload_log
        #   run: |
        #     LOG_CONTENT=$(cat eat_by_date.log)
        #     JSON_PAYLOAD="{\"description\": \"Error Log for Eat by date\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
        #     GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
        #     echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #     - name: Set current date (Eat by Date)
  #       run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        # - name: Determine Color
        # id: set_color
        # run: |
        #   if [ "${{ job.status }}" == "success" ]; then
        #     echo "::set-output name=color::#36a64f"
        #   else
        #     echo "::set-output name=color::#ff0000"
        #   fi

  #     - name: Notify Slack (Eat by Date Script)
  #       if: always()
  #       uses: 8398a7/action-slack@v3
  #       with:
  #         status: custom
  #         custom_payload: |
  #           {
  #             "attachments": [
  #               {
  #                 "color": "${{ steps.set_color.outputs.color }}",
  #                 "text": "Eat by Date Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #               }
  #             ]
  #           }
  #       env:
  #         SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}

# run_flipkart:
#     runs-on: ubuntu-latest
#     steps:
#       - uses: actions/checkout@v2

#       - name: Setup Python
#         uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - name: Cache Python packages
#         uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: |
#             ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: python -m pip install --upgrade pip && pip install -r requirements.txt

#       - name: Execute scrap_flipkart_pages script
#         run: python scrap_flipkart_pages_sel.py || echo "Script failed, check logs." > flipkart_error.log
#         working-directory: ./landing_zone/collectors/Flipkart
#         continue-on-error: true

#       - name: Prepare error log (flipkart)
#         if: failure()
#         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && flipkart_error.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
#         working-directory: ./landing_zone/collectors/Flipkart

#       - name: Upload error log to Gist (flipkart)
#         if: failure()
#         id: upload_log
#         run: |
#           LOG_CONTENT=$(cat flipkart_error.log)
#           JSON_PAYLOAD="{\"description\": \"Error Log for flipkat\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
#           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
#           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

#       - name: Determine success/failure indicator and message (Mealdb)
#         id: status_indicator
#         run: |
#           if [ -s flipkart_error.log ]; then
#             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
#             echo "COLOR=#ff0000" >> $GITHUB_ENV
#           else
#             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
#             echo "COLOR=#36a64f" >> $GITHUB_ENV
#           fi
#         working-directory: ./landing_zone/collectors/Flipkart

        # - name: Determine Color
        #   id: set_color
        #   run: |
        #     if [ "${{ job.status }}" == "success" ]; then
        #       echo "::set-output name=color::#36a64f"
        #     else
        #       echo "::set-output name=color::#ff0000"
        #     fi


#       - name: Set current date (flipkart)
#         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

#       - name: Notify Slack (flipkart Script)
#         if: always()
#         uses: 8398a7/action-slack@v3
#         with:
#           status: custom
#           custom_payload: |
#             {
#               "attachments": [
#                 {
#                   "color": "${{ steps.set_color.outputs.color }}",
#                   "text": "Flipkart Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
#                 }
#               ]
#             }
#         env:
#           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"


  run_big_basket:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute bigBasket script
          run: python big_basket.py || echo "Script failed, check logs." > bigBasket.log
          working-directory: ./landing_zone/collectors/big_basket
          # continue-on-error: false

        - name: Prepare error log (bigBasket)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat bigBasket.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/collectors/big_basket

        - name: Upload error log to Gist (bigbasket)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat bigBasket.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for Bigbasket\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (bigBasket)
          id: status_indicator
          run: |
            if [ -s bigBasket.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/collectors/big_basket

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi

        - name: Set current date (bigBasket)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (bigBasket Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "BigBasket Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  # run_catalonia_establishment_locationt:
  #     runs-on: ubuntu-latest
  #     steps:
  #       - uses: actions/checkout@v2

  #       - name: Setup Python
  #         uses: actions/setup-python@v4
  #         with:
  #           python-version: '3.10'

  #       - name: Cache Python packages
  #         uses: actions/cache@v3
  #         with:
  #           path: ~/.cache/pip
  #           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #           restore-keys: |
  #             ${{ runner.os }}-pip-

  #       - name: Install dependencies
  #         run: python -m pip install --upgrade pip && pip install -r requirements.txt

  #       - name: Execute catalonia_establishment_location script
  #         run: python establishments_scraper.py || echo "Script failed, check logs." > catalonia_establishment_location.log
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location
  #         continue-on-error: true

  #       - name: Prepare error log (establishment location)
  #         if: failure()
  #         run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat catalonia_establishment_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location

  #       - name: Upload error log to Gist (Establishment Location)
  #         if: failure()
  #         id: upload_log
  #         run: |
  #           LOG_CONTENT=$(cat catalonia_establishment_location.log)
  #           JSON_PAYLOAD="{\"description\": \"Error Log for Establishment Location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
  #           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
  #           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

  #       - name: Determine success/failure indicator and message (catalonia_establishment_location)
  #         id: status_indicator
  #         run: |
  #           if [ -s catalonia_establishment_location.log ]; then
  #             echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
  #             echo "COLOR=#ff0000" >> $GITHUB_ENV
  #           else
  #             echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
  #             echo "COLOR=#36a64f" >> $GITHUB_ENV
  #           fi
  #         working-directory: ./landing_zone/collectors/catalonia_establishment_location

  #       - name: Determine Color
  #         id: set_color
  #         run: |
  #           if [ "${{ job.status }}" == "success" ]; then
  #             echo "::set-output name=color::#36a64f"
  #           else
  #             echo "::set-output name=color::#ff0000"
  #           fi


  #       - name: Set current date (catalonia_establishment_location)
  #         run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #       - name: Notify Slack (catalonia_establishment_location Script)
  #         if: always()
  #         uses: 8398a7/action-slack@v3
  #         with:
  #           status: custom
  #           custom_payload: |
  #             {
  #               "attachments": [
  #                 {
  #                   "color": "${{ steps.set_color.outputs.color }}",
  #                   "text": "Catalonia Establishment Location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #                 }
  #               ]
  #             }
  #         env:
  #           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"




  run_OCR:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute OCR script
          run: python ocr_marcadona.py || echo "Script failed, check logs." > ocr.log
          working-directory: ./landing_zone/collectors/OCR
          # continue-on-error: false

        - name: Prepare error log (OCR)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat ocr.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/collectors/OCR

        - name: Upload error log to Gist (ocr)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat ocr.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for ocr\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (OCR)
          id: status_indicator
          run: |
            if [ -s OCR.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/collectors/OCR

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi

        - name: Set current date (OCR)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (OCR Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "OCR Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  run_customers:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute customer script
          run: python sm_retail_customer.py || echo "Script failed, check logs." > customer.log
          working-directory: ./landing_zone/collectors/customers
          # continue-on-error: false

        - name: Prepare error log (customers)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customers.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/collectors/customers

        - name: Upload error log to Gist (customers)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat customers.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for customers\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (customers)
          id: status_indicator
          run: |
            if [ -s customers.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/collectors/customers

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi


        - name: Set current date (customers)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (customers Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "Customers Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  run_customer_location:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute customer_location script
          run: python customer_location.py || echo "Script failed, check logs." > customer_location.log
          working-directory: ./landing_zone/synthetic/customer_location
          # continue-on-error: false

        - name: Prepare error log (customer_location)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_location.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/synthetic/customer_location

        - name: Upload error log to Gist (customer location)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat customer_location.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for customer location\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (customer_location)
          id: status_indicator
          run: |
            if [ -s customer_location.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/synthetic/customer_location

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi


        - name: Set current date (customer_location)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (customer_location Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "Customer location Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"




  run_customer_purchase:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute customer_purchase script
          run: python customer_purchase.py || echo "Script failed, check logs." > customer_purchase.log
          working-directory: ./landing_zone/synthetic/customer_purchase
          # continue-on-error: false

        - name: Prepare error log (customer_purchase)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat customer_purchase.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/synthetic/customer_purchase

        - name: Upload error log to Gist (customer purchase)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat customer_purchase.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for customer purchase\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (customer_purchase)
          id: status_indicator
          run: |
            if [ -s customer_purchase.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/synthetic/customer_purchase

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi


        - name: Set current date (customer_purchase)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (customer_purchase Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "Customer purchase Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"



  run_sentiment_reviews:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute sentiment reviews script
          run: python business_reviews.py  || echo "Script failed, check logs." > sentiment_reviews.log
          working-directory: ./landing_zone/synthetic/sentiment reviews
          # continue-on-error: false

        - name: Execute sentiment reviews script
          run: python individual_reviews.py  || echo "Script failed, check logs." > sentiment_reviews.log
          working-directory: ./landing_zone/synthetic/sentiment reviews
          # continue-on-error: false

        - name: Prepare error log (sentiment reviews)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat sentiment_reviews.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/synthetic/sentiment reviews

        - name: Upload error log to Gist (sentiment_reviews)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat sentiment_reviews.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for sentiment reviews\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (sentiment reviews)
          id: status_indicator
          run: |
            if [ -s sentiment_reviews.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/synthetic/sentiment reviews

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi


        - name: Set current date (sentiment reviews)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (sentiment reviews Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "Sentiment reviews Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"

  run_supermarket_products:
      runs-on: ubuntu-latest
      steps:
        - uses: actions/checkout@v2

        - name: Setup Python
          uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Cache Python packages
          uses: actions/cache@v3
          with:
            path: ~/.cache/pip
            key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
            restore-keys: |
              ${{ runner.os }}-pip-

        - name: Install dependencies
          run: python -m pip install --upgrade pip && pip install -r requirements.txt

        - name: Execute supermarket_products script
          run: python supermarket_products.py || echo "Script failed, check logs." > supermarket_products.log
          working-directory: ./landing_zone/synthetic/supermarket_products
          # continue-on-error: false

        - name: Prepare error log (supermarket_products)
          if: failure()
          run: echo "ERROR_LOG<<EOF" >> $GITHUB_ENV && cat supermarket_products.log >> $GITHUB_ENV && echo "EOF" >> $GITHUB_ENV
          working-directory: ./landing_zone/synthetic/supermarket_products

        - name: Upload error log to Gist (supermarket_products)
          if: failure()
          id: upload_log
          run: |
            LOG_CONTENT=$(cat supermarket_products.log)
            JSON_PAYLOAD="{\"description\": \"Error Log for supermarket products\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
            GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
            echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

        - name: Determine success/failure indicator and message (supermarket_products)
          id: status_indicator
          run: |
            if [ -s supermarket_products.log ]; then
              echo "STATUS_MESSAGE=Fail :x:" >> $GITHUB_ENV
              echo "COLOR=#ff0000" >> $GITHUB_ENV
            else
              echo "STATUS_MESSAGE=Success :white_check_mark:" >> $GITHUB_ENV
              echo "COLOR=#36a64f" >> $GITHUB_ENV
            fi
          working-directory: ./landing_zone/synthetic/supermarket_products

        - name: Determine Color
          id: set_color
          run: |
            if [ "${{ job.status }}" == "success" ]; then
              echo "::set-output name=color::#36a64f"
            else
              echo "::set-output name=color::#ff0000"
            fi



        - name: Set current date (supermarket_products)
          run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

        - name: Notify Slack (supermarket_products Script)
          if: always()
          uses: 8398a7/action-slack@v3
          with:
            status: custom
            custom_payload: |
              {
                "attachments": [
                  {
                    "color": "${{ steps.set_color.outputs.color }}",
                    "text": "Supermarket products Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View url\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
                  }
                ]
              }
          env:
            SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"


# name: project.py

# on:
#   schedule:
#     - cron: '0 0 * * 5' # At 00:00 on Friday
#   push:
#     branches:
#       - main

# jobs:
#   run_scripts:
#     runs-on: ubuntu-latest
#     strategy:
#       matrix:
#         include:
#           - name: 'ocr'
#             path: 'OCR'
#             command: 'python ocr_marcadona.py'
#           - name: 'customers'
#             path: 'customers'
#             command: 'python sm_retail_customer.py'
#           # - name: 'catalonia_establishment_location'
#             # path: 'catalonia_establishment_location'
#             # command: 'python establishments_scraper.py'
#           # - name: 'flipkart'
#           #   path: 'Flipkart'
#           #   command: 'python scrap_flipkart_pages_sel.py'
#           # - name: 'Eatbydate'
#           #   path: 'eat_by_date'
#           #   command: 'python eat_by_date.py'
#           - name: 'Mealdb'
#             path: 'meal_db/mealscraper'
#             command: 'scrapy crawl mealdb -O ./data/raw/mealdb.json'
#           - name: 'ApprovedFood'
#             path: 'approved_food_uk/approvedfood_scraper'
#             command: 'scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json'
#           - name: 'BigBasket'
#             path: 'big_basket'
#             command: 'python big_basket.py'
#           - name: 'CustomerLocation'
#             path: 'synthetic/customer_location'
#             command: 'python customer_location.py'
#           - name: 'CustomerPurchase'
#             path: 'synthetic/customer_purchase'
#             command: 'python customer_purchase.py'
#           - name: 'Business_SentimentReview'
#             path: 'synthetic/sentiment reviews'
#             command: 'python business_reviews.py'
#           - name: 'Individual_SentimentReview'
#             path: 'synthetic/sentiment reviews'
#             command: 'python individual_reviews.py'
#           - name: 'SupermarketProducts'
#             path: 'synthetic/supermarket_products'
#             command: 'python supermarket_products.py'

#     steps:
#       - uses: actions/checkout@v2

#       - name: Setup Python
#         uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - name: Cache Python packages
#         uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: python -m pip install --upgrade pip && pip install -r requirements.txt

#       - name: Execute Script
#         run: |
#           cd ./landing_zone/collectors/${{ matrix.path }}
#           ${{ matrix.command }} 
         
#         continue-on-error: true

#       - name: Determine Color
#         id: set color
#         run: |
#             if [ "${{ job.status }}" == "success" ]; then
#               echo "::set-output name=color::#36a64f"
#             else
#               echo "::set-output name=color::#ff0000"
#             fi

#       - name: Prepare error log
#         if: failure()
#         run: |
#           echo "ERROR_LOG<<EOF" >> $GITHUB_ENV
#           cat ./logs/${{ matrix.name }}_error.log >> $GITHUB_ENV
#           echo "EOF" >> $GITHUB_ENV

#       - name: Upload error log to Gist
#         if: failure()
#         run: |
#           LOG_CONTENT=$(cat ./logs/${{ matrix.name }}_error.log)
#           JSON_PAYLOAD="{\"description\": \"Error Log for ${{ matrix.name }}\", \"public\": true, \"files\": {\"error_log.txt\": {\"content\": \"$LOG_CONTENT\"}}}"
#           GIST_URL=$(curl -X POST -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d "$JSON_PAYLOAD" https://api.github.com/gists | jq -r '.html_url')
#           echo "GIST_URL=$GIST_URL" >> $GITHUB_ENV

#       - name: Notify Slack
#         if: always()
#         uses: 8398a7/action-slack@v3
#         with:
#           status: custom
#           custom_payload: |
#             {
#               "attachments": [
#                 {
#                   "color": "${{ steps.set_color.outputs.color }}",
#                   "text": "${{ matrix.name }} Script\nStatus: ${{ job.status }}\nError Log: <${{ env.GIST_URL }}|View Log>\nDate: ${{ env.CURRENT_DATE }}\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
#                 }
#               ]
#             }
#         env:
#           SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
  
  
  
  



  # name: project.py

# on:
#   schedule:
#     - cron: '0 0 * * 5' # At 00:00 on Friday
#   push:
#     branches:
#       - main

# jobs:
#   run_Mealdb:
#     runs-on: ubuntu-latest
#     outputs:
#       script_name: 'Mealdb Script'
#       status: ${{ steps.set_status.outputs.status }}
#       error: ${{ steps.set_status.outputs.error }}
#     steps:
#       - uses: actions/checkout@v2

#       - uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'

#       - uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#           restore-keys: |
#             ${{ runner.os }}-pip-

#       - name: Install dependencies
#         run: |
#           python -m pip install --upgrade pip
#           pip install -r requirements.txt

#       - name: Execute mealdb Scrapy spider
#         id: mealdb_execution
#         run: |
#           scrapy crawl mealdb -O ./data/raw/mealdb.json . 2>mealdb_error.log || echo "error" > mealdb_error.log
#         working-directory: ./landing_zone/collectors/Mealdb/mealscraper
      
#       - name: Set job status and error
#         id: set_status
#         if: always()
#         run: |
#           if [ -s mealdb_error.log ]; then
#             ERROR_MSG=$(<mealdb_error.log)
#             echo "::set-output name=status::failure"
#             echo "::set-output name=error::$ERROR_MSG"
#           else
#             echo "::set-output name=status::success"
#             echo "::set-output name=error::None"
#           fi

#   run_ApprovedFood:
#     runs-on: ubuntu-latest
#     outputs:
#       script_name: 'ApprovedFood Script'
#       status: ${{ steps.set_status.outputs.status }}
#       error: ${{ steps.set_status.outputs.error }}
#     steps:
#       - uses: actions/checkout@v2
#       - uses: actions/setup-python@v4
#         with:
#           python-version: '3.10'
#       - uses: actions/cache@v3
#         with:
#           path: ~/.cache/pip
#           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
#       - name: Install dependencies
#         run: |
#           python -m pip install --upgrade pip
#           pip install -r requirements.txt
#       - name: Execute approvedfood Scrapy spider
#         run: |
#           scrapy crawl approvedfood_groceries -O ./data/raw/Approvedfood.json 2>approvedfood_error.log || echo "error" > approvedfood_error.log
#         working-directory: ./landing_zone/collectors/ApprovedFoodUK/approvedfood_scraper
#       - name: Set job status and error
#         id: set_status
#         if: always()
#         run: |
#           if [ -s approvedfood_error.log ]; then
#             ERROR_MSG=$(<approvedfood_error.log)
#             echo "::set-output name=status::failure"
#             echo "::set-output name=error::$ERROR_MSG"
#           else
#             echo "::set-output name=status::success"
#             echo "::set-output name=error::None"
#           fi

  # run_eat_by_date:
  #   runs-on: ubuntu-latest
  #   outputs:
  #     script_name: 'Eat by Date Script'
  #     status: ${{ steps.set_status.outputs.status }}
  #     error: ${{ steps.set_status.outputs.error }}
  #   steps:
  #     - uses: actions/checkout@v2
  #     - uses: actions/setup-python@v4
  #       with:
  #         python-version: '3.10'
  #     - uses: actions/cache@v3
  #       with:
  #         path: ~/.cache/pip
  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
  #     - name: Install dependencies
  #       run: |
  #         python -m pip install --upgrade pip
  #         pip install -r requirements.txt
  #     - name: Execute eat_by_date.py script
  #       run: |
  #         python eat_by_date.py 2>eat_by_date_error.log || echo "error" > eat_by_date_error.log
  #       working-directory: ./landing_zone/collectors/eat_by_date
  #     - name: Set job status and error
  #       id: set_status
  #       if: always()
  #       run: |
  #         if [ -s eat_by_date_error.log ]; then
  #           ERROR_MSG=$(<eat_by_date_error.log)
  #           echo "::set-output name=status::failure"
  #           echo "::set-output name=error::$ERROR_MSG"
  #         else
  #           echo "::set-output name=status::success"
  #           echo "::set-output name=error::None"
  #         fi

  # notify_slack:
  #   needs: [run_Mealdb, run_ApprovedFood] #, run_eat_by_date]
  #   runs-on: ubuntu-latest
  #   steps:
  #     - name: Set current date
  #       run: echo "CURRENT_DATE=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_ENV

  #     - name: Notify Slack on Error
  #       uses: 8398a7/action-slack@v3
  #       with:
  #         status: custom
  #         custom_payload: |
  #           {
  #             "attachments": [{
  #               "color": "${{ needs.run_Mealdb.outputs.status }} === 'success' && '${{ needs.run_ApprovedFood.outputs.status }} === 'success' ? 'good' : 'danger'", 
  #               "text": "Workflow: ${{ github.workflow }}\nDate: ${{ env.CURRENT_DATE }}\nJob: Detailed Status Update\n- Mealdb Script Status: ${{ needs.run_Mealdb.outputs.status }} (Error: ${{ needs.run_Mealdb.outputs.error }})\n- ApprovedFood Script Status: ${{ needs.run_ApprovedFood.outputs.status }} (Error: ${{ needs.run_ApprovedFood.outputs.error }})\nCommit: ${{ github.sha }}\nAuthor: ${{ github.actor }}\nRepo: ${{ github.repository }}"
  #             }]
  #           }
  #       env:
  #         SLACK_WEBHOOK_URL: "https://hooks.slack.com/services/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"
  #         CURRENT_DATE: ${{ env.CURRENT_DATE }}
  #       if: always()