2402 uptime robot monitoring (#644)

* feat: get all urls with http methods

* feat: create monitors for unmonitored endpoints in uptime robot

* feat: output json file

* feat: load secrets in ci

* fix: config one password

* fix: unneeded param

* debug print

* fix: loading uptime api key

* fix: load secrets

* fix: configure one password

* feat: return openapi schema

* feat: testing the approach of a config for the API endpoints

* fix: configuration of monitored urls, proper failing of command, existing monitors are deleted and re-created

* feat: adding command to generate JWT token for api monitoring

* feat: added config also for the ceramic-cache namespace, remove redundant command to generate test JWT token

* feat: remove unused action, update workflows for the new uptime robot monitoring script

* fix: remove unused workflows

* fix: adjust workflows, to do a dry-run on the uptimerobot script unless deploying to prod

* feat: skipped URLs need to be present in uptime-robot (they have to be manually set up), otherwise the command will fail

* fix: remove unnecesarry check & setting of expected status codes

---------

Co-authored-by: Gerald Iakobinyi-Pich <nutrina9@gmail.com>
Co-authored-by: Gerald Iakobinyi-Pich <gerald@gitcoin.co>

.github/workflows/build_and_deploy_review.yml

@@ -32,6 +32,7 @@ jobs:
     with:
       refspec: ${{ needs.ref.outputs.refspec }}
       environment: review
+      uptime-robot-monitor-dry-run: --dry-run
     secrets: inherit
 
   deploy:

.github/workflows/build_and_deploy_staging.yml

@@ -29,6 +29,7 @@ jobs:
     with:
       refspec: ${{ needs.ref.outputs.refspec }}
       environment: review
+      uptime-robot-monitor-dry-run: --dry-run
     secrets: inherit
 
   deploy:

.github/workflows/ci.yml

@@ -71,6 +71,48 @@ jobs:
           DATA_MODEL_DATABASE_URL: postgres://passport_scorer:passport_scorer_pwd@localhost:5432/passport_scorer
           FF_API_ANALYTICS: on
 
+      - name: Install 1Password CLI
+        uses: 1password/install-cli-action@v1
+
+      - name: Configure 1Password Service Account
+        uses: 1password/load-secrets-action/configure@v1
+        with:
+          service-account-token: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+
+      - name: Load secret
+        id: op-load-secret
+        uses: 1password/load-secrets-action@v1
+        with:
+          export-env: true
+        env:
+          OP_SEVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
+
+          UPTIME_ROBOT_API_KEY: op://DevOps/uptime-robot-config/ci/UPTIME_ROBOT_API_KEY
+          REGISTRY_API_KEY: op://DevOps/uptime-robot-config/ci/REGISTRY_API_KEY
+          REGISTRY_ROUND_ID: op://DevOps/uptime-robot-config/ci/REGISTRY_ROUND_ID
+          REGISTRY_SCORER_ID: op://DevOps/uptime-robot-config/ci/REGISTRY_SCORER_ID
+          REGISTRY_ADDRESS: op://DevOps/uptime-robot-config/ci/REGISTRY_ADDRESS
+          CERAMIC_CACHE_JWT_TOKEN: op://DevOps/uptime-robot-config/ceramic-cache/JWT_TOKEN
+          CERAMIC_CACHE_ADDRESS: op://DevOps/uptime-robot-config/ceramic-cache/ADDRESS
+
+      - name: URL Monitoring Check
+        working-directory: ./api
+        shell: bash
+        env:
+          CERAMIC_CACHE_SCORER_ID: ""
+          SECRET_KEY: secret-test-value
+          UPTIME_ROBOT_API_KEY: ${{ env.UPTIME_ROBOT_API_KEY }}
+
+          REGISTRY_API_KEY: ${{ env.REGISTRY_API_KEY }}
+          REGISTRY_ROUND_ID: ${{ env.REGISTRY_ROUND_ID }}
+          REGISTRY_SCORER_ID: ${{ env.REGISTRY_SCORER_ID }}
+          REGISTRY_ADDRESS: ${{ env.REGISTRY_ADDRESS }}
+          CERAMIC_CACHE_JWT_TOKEN: ${{ env.CERAMIC_CACHE_JWT_TOKEN }}
+          CERAMIC_CACHE_ADDRESS: ${{ env.CERAMIC_CACHE_ADDRESS }}
+
+        run:
+          python manage.py get_unmonitored_urls --base-url https://api.scorer.gitcoin.co/
+
   ui-test:
     runs-on: ubuntu-latest
 

.github/workflows/test_generic.yml

@@ -10,9 +10,16 @@ on:
         description: "Environment to deploy to"
         type: string
         required: true
+      uptime-robot-monitor-dry-run:
+        type: choice
+        description: "Set to '--dry-run' to not actually create monitors"
+        default: ""
+        required: false
+        options:
+          - ""
+          - "--dry-run"
 
 jobs:
-
   test:
     runs-on: ubuntu-latest
     services:
@@ -80,17 +87,30 @@ jobs:
           IGNORE_UNMONITORED_URLS: op://DevOps/passport-scorer-api-review-env/ci/IGNORE_UNMONITORED_URLS
           OP_SERVICE_ACCOUNT_TOKEN: ${{ secrets.OP_SERVICE_ACCOUNT_TOKEN }}
 
+          UPTIME_ROBOT_API_KEY: op://DevOps/uptime-robot-config/ci/UPTIME_ROBOT_API_KEY
+          REGISTRY_API_KEY: op://DevOps/uptime-robot-config/ci/REGISTRY_API_KEY
+          REGISTRY_ROUND_ID: op://DevOps/uptime-robot-config/ci/REGISTRY_ROUND_ID
+          REGISTRY_SCORER_ID: op://DevOps/uptime-robot-config/ci/REGISTRY_SCORER_ID
+          REGISTRY_ADDRESS: op://DevOps/uptime-robot-config/ci/REGISTRY_ADDRESS
+          CERAMIC_CACHE_JWT_TOKEN: op://DevOps/uptime-robot-config/ceramic-cache/JWT_TOKEN
+          CERAMIC_CACHE_ADDRESS: op://DevOps/uptime-robot-config/ceramic-cache/ADDRESS
+
       - name: URL Monitoring Check
         working-directory: ./api
         shell: bash
         env:
           CERAMIC_CACHE_SCORER_ID: ""
           SECRET_KEY: secret-test-value
-          UPTIME_ROBOT_READONLY_API_KEY: ${{ env.UPTIME_ROBOT_READONLY_API_KEY }}
-          IGNORE_UNMONITORED_URLS: ${{ env.IGNORE_UNMONITORED_URLS }}
-        run: python manage.py show_urls -f json > urls.json &&
-          python manage.py get_unmonitored_urls --urls urls.json --base-url https://api.scorer.gitcoin.co --out unmonitored.json --allow-paused True &&
-          [ -f unmonitored.json ] && [ `cat unmonitored.json | wc -m` -eq 2 ]
+          UPTIME_ROBOT_API_KEY: ${{ env.UPTIME_ROBOT_API_KEY }}
+
+          REGISTRY_API_KEY: ${{ env.REGISTRY_API_KEY }}
+          REGISTRY_ROUND_ID: ${{ env.REGISTRY_ROUND_ID }}
+          REGISTRY_SCORER_ID: ${{ env.REGISTRY_SCORER_ID }}
+          REGISTRY_ADDRESS: ${{ env.REGISTRY_ADDRESS }}
+          CERAMIC_CACHE_JWT_TOKEN: ${{ env.CERAMIC_CACHE_JWT_TOKEN }}
+          CERAMIC_CACHE_ADDRESS: ${{ env.CERAMIC_CACHE_ADDRESS }}
+
+        run: python manage.py get_unmonitored_urls --base-url https://api.scorer.gitcoin.co/ ${{ inputs.uptime-robot-monitor-dry-run }}
 
       - name: Run API unittests
         working-directory: ./api

api/.env-sample

@@ -49,7 +49,7 @@ CGRANTS_API_TOKEN=abc
 REGISTRY_API_READ_DB=default
 
 STAKING_SUBGRAPH_API_KEY=abc
-UPTIME_ROBOT_READONLY_API_KEY=abc
+UPTIME_ROBOT_API_KEY=abc
 
 ETHEREUM_MODEL_ENDPOINT=http://localhost:8123/eth-stamp-predict
 NFT_MODEL_ENDPOINT=http://localhost:8123/eth-nft-predict

api/registry/management/commands/generate_ceramiccache_access_token.py

@@ -1,13 +1,10 @@
 from datetime import timedelta
+
 from django.core.management.base import BaseCommand
 
 from ceramic_cache.api.v1 import DbCacheToken
 
 
-class LongLivedToken(DbCacheToken):
-    lifetime: timedelta = timedelta(days=100 * 365)
-
-
 class Command(BaseCommand):
     help = "Removes stamp data and sets score to 0 for users in the provided list"
 
@@ -23,7 +20,8 @@ def handle(self, *args, **kwargs):
         address = kwargs["address"].lower()
         self.stdout.write(f"Generating long-lived access token for {address}")
 
-        token = LongLivedToken()
+        token = DbCacheToken()
+        token.access_token_class.lifetime = timedelta(days=356)
         token["did"] = f"did:pkh:eip155:1:{address}"
 
         self.stdout.write("Access token:")