Add workflow and script to periodically check for certs expiry (#589)

* Add workflow and script to periodically check for certs expiry * Add debug comments * Remove debug commands * Add workflow and script for running cron check certs * Install envsubst * Remove debug lines * Update cron script
airtai · Apr 2, 2024 · a9cc220 · a9cc220
1 parent 6e35fe5
commit a9cc220
Show file tree

Hide file tree

Showing 4 changed files with 167 additions and 0 deletions.
diff --git a/.github/workflows/check_certs_expiry.yml b/.github/workflows/check_certs_expiry.yml
@@ -0,0 +1,55 @@
+name: Check letsencrypt certs expiry
+
+on:
+  # everyday at 00:00
+  schedule:
+    - cron: "0 0 * * *"
+  # or manually
+  workflow_dispatch:
+
+jobs:
+  check-certs-expiry:
+    name: Check letsencrypt certs expiry
+    strategy:
+      fail-fast: false
+      matrix:
+        domain:
+          [
+            "backend.captn.ai",
+            "staging.backend.captn.ai",
+            "auth.captn.ai",
+            "staging.auth.captn.ai",
+          ]
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+    container:
+      image: ubuntu:22.04
+    env:
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+      # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989
+      - run: chown -R $(id -u):$(id -g) $PWD
+
+      - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV
+      - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git -y )"
+      - run: apt-get update -y && apt-get install sshpass -y
+      - run: eval $(ssh-agent -s)
+      - run: mkdir -p ~/.ssh
+      - run: chmod 700 ~/.ssh
+
+      - run: echo "DOMAIN=${{ matrix.domain }}" >> $GITHUB_ENV
+
+      - run: ssh-keyscan "$DOMAIN" >> ~/.ssh/known_hosts
+      - run: chmod 644 ~/.ssh/known_hosts
+
+      - run: if [[ $DOMAIN == *"staging"* ]]; then echo "$STAGING_SSH_KEY" | base64 --decode > key.pem ; else echo "$PROD_SSH_KEY" | base64 --decode > key.pem ; fi;
+      - run: chmod 600 key.pem
+
+      - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "ls -la"
+      - run: bash ./scripts/ci_check_certs.sh
+
+      - run: rm key.pem
diff --git a/.github/workflows/update_cron_script.yml b/.github/workflows/update_cron_script.yml
@@ -0,0 +1,62 @@
+name: Update cron script when changes are made
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'scripts/ci_check_certs.sh'
+      - '.github/workflows/update_cron_script.yml'
+  workflow_dispatch:
+
+jobs:
+  check-certs-expiry:
+    name: Update cron script when changes are made
+    strategy:
+      fail-fast: false
+      matrix:
+        domain:
+          [
+            "backend.captn.ai",
+            "staging.backend.captn.ai",
+            "auth.captn.ai",
+            "staging.auth.captn.ai",
+          ]
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash
+    container:
+      image: ubuntu:22.04
+    env:
+      PROD_SSH_KEY: ${{ secrets.PROD_SSH_KEY }}
+      STAGING_SSH_KEY: ${{ secrets.STAGING_SSH_KEY }}
+    steps:
+      - uses: actions/checkout@v3
+      # This is to fix GIT not liking owner of the checkout dir - https://github.com/actions/runner/issues/2033#issuecomment-1204205989
+      - run: chown -R $(id -u):$(id -g) $PWD
+
+      - run: echo "PATH=$PATH:/github/home/.local/bin" >> $GITHUB_ENV
+      # Installing gettext to use envsubst command
+      - run: "which ssh-agent || ( apt-get update -y && apt-get install openssh-client git gettext -y )"
+      - run: apt-get update -y && apt-get install sshpass -y
+      - run: eval $(ssh-agent -s)
+      - run: mkdir -p ~/.ssh
+      - run: chmod 700 ~/.ssh
+
+      - run: echo "DOMAIN=${{ matrix.domain }}" >> $GITHUB_ENV
+
+      - run: ssh-keyscan "$DOMAIN" >> ~/.ssh/known_hosts
+      - run: chmod 644 ~/.ssh/known_hosts
+
+      - run: if [[ $DOMAIN == *"staging"* ]]; then echo "$STAGING_SSH_KEY" | base64 --decode > key.pem ; else echo "$PROD_SSH_KEY" | base64 --decode > key.pem ; fi;
+      - run: chmod 600 key.pem
+
+      - run: ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "ls -la"
+
+      - run: envsubst '${DOMAIN}' < scripts/cron_check_certs.sh > tmp.sh
+      - run: chmod +x tmp.sh
+      - run: cat tmp.sh
+      - run: scp -i key.pem tmp.sh azureuser@"$DOMAIN":/home/azureuser/cron_check_certs.sh
+
+      - run: rm key.pem
diff --git a/scripts/ci_check_certs.sh b/scripts/ci_check_certs.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# Set the threshold for certificate expiration (in days)
+THRESHOLD=15
+
+# Check if certificate is expiring within the threshold
+expiry_str=$(ssh -o StrictHostKeyChecking=no -i key.pem azureuser@"$DOMAIN" "sudo openssl x509 -in /etc/letsencrypt/live/$DOMAIN/fullchain.pem -noout -enddate | cut -d= -f2")
+expiration_date=$(date -d "$expiry_str" +%s)
+current_date=$(date +%s)
+days_until_expiry=$(( (expiration_date - current_date) / 86400 ))
+
+if [ "$days_until_expiry" -le "$THRESHOLD" ]; then
+    echo "Certificate for $DOMAIN is expiring in $days_until_expiry days. PLEASE RENEW ASAP."
+    exit -1
+else
+    echo "Certificate for $DOMAIN is not expiring within the next $THRESHOLD days. Expires after $days_until_expiry days. No action needed."
+fi
diff --git a/scripts/cron_check_certs.sh b/scripts/cron_check_certs.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Set the threshold for certificate expiration (in days)
+THRESHOLD=30
+
+# List of domains for which to check and renew certificates
+DOMAINS=("$DOMAIN")
+
+echo "Stopping nginx"
+sudo service nginx stop
+
+for domain in "${DOMAINS[@]}"; do
+    # Check if certificate is expiring within the threshold
+    expiration_date=$(date -d "$(sudo openssl x509 -in /etc/letsencrypt/live/$domain/fullchain.pem -noout -enddate | cut -d= -f2)" +%s)
+    current_date=$(date +%s)
+    days_until_expiry=$(( (expiration_date - current_date) / 86400 ))
+
+    if [ "$days_until_expiry" -lt "$THRESHOLD" ]; then
+        echo "Certificate for $domain is expiring in $days_until_expiry days. Renewing..."
+        # sudo certbot renew --force-renewal
+        sudo certbot renew --cert-name "$domain"
+        if [ $? -eq 0 ]; then
+            echo "Certificate renewed successfully for $domain."
+        else
+            echo "Certificate renewal failed for $domain."
+        fi
+    else
+        echo "Certificate for $domain is not expiring within the next $THRESHOLD days, expiring after $days_until_expiry days. No action needed."
+    fi
+done
+
+echo "Restarting nginx"
+sudo service nginx restart