Merge pull request #285 from caktus/CU-8689pdzrr-k8s-self-hosted-runner

K8s Self-hosted GitHub Runner

Author: ronardcaktus

.github/workflows/deploy.yml

@@ -7,10 +7,8 @@ on:
       - develop
 
 jobs:
-  deploy:
-    runs-on:
-      - self-hosted
-      - philly-hip
+  build-push:
+    runs-on: ubuntu-22.04 # standard (not self-hosted) runner
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
@@ -23,24 +21,82 @@ jobs:
         with:
           python-version: '3.10'
           cache: 'pip'
-          cache-dependency-path: 'requirements/*/**.txt'
+          cache-dependency-path: 'requirements/*/dev.txt'
       - name: Install dependencies
         id: pip-install
         run: |
           python -m pip install --upgrade pip wheel pip-tools
-          pip-sync requirements/base/base.txt requirements/dev/dev.txt
+          pip-sync requirements/dev/dev.txt
       - name: Login to Docker
         id: docker-login
         run: |
           inv aws.docker-login
-      - name: Build, tag, push, and deploy image
-        id: build-tag-push-deploy
+      - name: Set DOCKER_TAG and save to file for artifact upload
+        run: |
+          DOCKER_TAG=$(inv image.tag | grep 'Set config.tag to' | cut -d' ' -f4)
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+          echo "$DOCKER_TAG" > docker-tag.txt
+      - name: Build, tag, and push image
+        id: build-tag-push
         run: |
           [ "$GITHUB_REF" = refs/heads/main ] &&
             ENV="production" ||
             ENV="staging"
           echo "env is $ENV"
-          inv $ENV image deploy --verbosity=0
+          export BUILDKIT_PROGRESS=plain
+          inv $ENV image.build --tag=${{ env.DOCKER_TAG }} image.push --tag=${{ env.DOCKER_TAG }}
+      - name: Upload docker tag from build-push job
+        uses: actions/upload-artifact@v4
+        with:
+          name: docker_tag
+          path: docker-tag.txt
+
+  # The deploy needs to run from within the cluster, since the cluster
+  # is not exposed to the public internet. This step is split out into
+  # its own job to reduce the amount of work done on the self-hosted runner
+  # and avoid the need to run a privileged docker container (with the
+  # capability of building a docker container itself).
+  deploy:
+    runs-on: arc-runner-set # K8s self-hosted runner
+    needs: [build-push]
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      # https://github.com/marketplace/actions/slack-github-actions-slack-integration
+      SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
+    steps:
+      - uses: actions/checkout@v3
+      - name: Download docker tag from build-push job
+        uses: actions/download-artifact@v4
+        with:
+          name: docker_tag
+      - name: Set variables
+        run: |
+          DOCKER_TAG=$(cat docker-tag.txt)
+          echo "DOCKER_TAG=$DOCKER_TAG" >> $GITHUB_ENV
+      - name: Install apt packages
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y git
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+          cache-dependency-path: 'requirements/*/dev.txt'
+      - name: Install dependencies
+        id: pip-install
+        run: |
+          python -m pip install --upgrade pip wheel pip-tools
+          pip-sync requirements/dev/dev.txt
+      - name: Deploy the image
+        id: deploy
+        run: |
+          [ "$GITHUB_REF" = refs/heads/main ] &&
+            ENV="production" ||
+            ENV="staging"
+          echo "env is $ENV" --verbosity=0
+          inv $ENV deploy --tag=${{ env.DOCKER_TAG }} --verbosity=0
       - uses: act10ns/slack@v1
         with:
           status: ${{ job.status }}

.github/workflows/test.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   test:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04 # standard (not self-hosted) runner
     env:
       DJANGO_SETTINGS_MODULE: hip.settings.dev
     services:

README.md

@@ -297,11 +297,11 @@ As mentioned in the Database setup instructions, you may need to visit
 
 ### GitHub Actions Runner
 
-There are [GitHub Actions self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) deployed on a virtual machine within the same VPC. Other runners may be added in the future, if needed.
+There are [GitHub Actions self-hosted runners](https://docs.github.com/en/actions/hosting-your-own-runners/about-self-hosted-runners) deployed in the Kubernetes cluster along side the application.
 
 Setup instructions:
 
-* Obtain a [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with the `admin:org` scope that's valid for one week (it needs to be active only for the initial deployment). Add this to a local environment variable `RUNNER_CFG_PAT`:
+* Obtain a [GitHub PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) with the `repo` scope that's valid for one week (it needs to be active only for the initial deployment). Add this to a local environment variable `RUNNER_CFG_PAT`:
 
 ```sh
 export RUNNER_CFG_PAT="gh......"
@@ -311,19 +311,5 @@ export RUNNER_CFG_PAT="gh......"
 
 ```sh
 cd deploy/
-# first time: connect as ubuntu user
-ansible-playbook -u ubuntu deploy-runner.yml
-# second time and beyond
 ansible-playbook deploy-runner.yml
 ```
-
-* The runner can be forcibly reinstalled by passing `-e force_reinstall=yes` or removed by passing `-e force_removal=yes`.
-
-Run OS updates:
-
-```sh
-cd deploy/
-ansible-playbook run-os-updates.yml
-# You may need to specify your username
-ansible-playbook -u myusername run-os-updates.yaml
-```

deploy/deploy-runner.yml

@@ -1,125 +1,47 @@
 ---
-- hosts: runner
-  become: yes
-  tags: base
-  roles:
-    - caktus.hosting_services.users
+- name: Install Actions Runner Controller and configure runner scale set
+  hosts: cluster
+  vars:
+    ansible_connection: local
+    ansible_python_interpreter: "{{ ansible_playbook_python }}"
+    runner_namespace: github-runner
+    chart_version: "0.9.3"
+  gather_facts: false
   tasks:
-    - name: Set hostname
-      hostname:
-        name: "{{ inventory_hostname_short }}"
-      when: inventory_hostname_short is defined and inventory_hostname_short
-    - name: Add new hostname to /etc/hosts
-      lineinfile:
-        path: /etc/hosts
-        regexp: '^127\.0\.1\.1'
-        line: '127.0.1.1 {{ inventory_hostname_short }}'
-        owner: root
-        group: root
-        mode: 0644
-      when: inventory_hostname_short is defined and inventory_hostname_short
+    # https://docs.github.com/en/actions/hosting-your-own-runners/managing-self-hosted-runners-with-actions-runner-controller/quickstart-for-actions-runner-controller
+    #
+    # Ansible task to automate:
+    # helm install arc \                      
+    # --namespace "${NAMESPACE}" \
+    # --create-namespace \
+    # oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
+    - name: Installing Actions Runner Controller
+      kubernetes.core.helm:
+        context: "{{ k8s_context|mandatory }}"
+        chart_ref: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set-controller
+        chart_version: "{{ chart_version }}"
+        release_name: arc
+        release_namespace: "{{ runner_namespace }}"
+        create_namespace: true
+        wait: yes
 
-- name: Install GitHub Actions Runner
-  hosts: runner
-  tags: runner
-  become: yes
-  tasks:
-    - name: Create runner user
-      ansible.builtin.user:
-        name: "{{ github_runner_user }}"
-        comment: Github Actions Runner
-    # Install Docker
-    # https://docs.docker.com/engine/install/ubuntu/#install-using-the-repository
-    - name: Install dependencies
-      ansible.builtin.package:
-        name:
-          - jq
-          - ca-certificates
-          - curl
-          - gnupg
-          - lsb-release
-          - libpq-dev
-          - python3.10
-          - python3.10-dev
-    - name: Add Docker's official GPG key
-      ansible.builtin.apt_key:
-        url: https://download.docker.com/linux/ubuntu/gpg
-        state: present
-    - name: Add Docker repository
-      ansible.builtin.apt_repository:
-        repo: deb [arch=amd64] https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable
-        state: present
-    - name: Install Docker Engine
-      ansible.builtin.package:
-        name:
-          - docker-ce
-          - docker-ce-cli
-          - containerd.io
-          - docker-buildx-plugin
-          - docker-compose-plugin
-        update_cache: yes
-    - name: Task name
-      stat:
-        path: /home/{{ github_runner_user }}/runner
-      register: runner_dir
-    - name: Set vars
-      set_fact:
-        run_removal_tasks: >-
-          {{
-            runner_dir.stat.exists
-            and (
-              (force_reinstall is defined and force_reinstall == "yes")
-              or (force_removal is defined and force_removal == "yes")
-            )
-          }}
-    # Various complicated Ansible roles exist, but this just works:
-    # https://github.com/actions/runner/blob/main/docs/automate.md
-    - name: Remove the runner
-      ansible.builtin.shell:
-        cmd: >
-          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/remove-svc.sh
-          | bash -s {{ github_scope }}
-        chdir: /home/{{ github_runner_user }}
-      environment:
-        RUNNER_CFG_PAT: "{{ github_pat }}"
-      when: run_removal_tasks
-      ignore_errors: True
-    - name: Delete the runner
-      ansible.builtin.shell:
-        cmd: >
-          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/delete.sh
-          | bash -s {{ github_scope }} {{ github_runner_name }}
-        chdir: /home/{{ github_runner_user }}
-      environment:
-        RUNNER_CFG_PAT: "{{ github_pat }}"
-      when: run_removal_tasks
-      ignore_errors: True
-    - name: Remove old runner directory and files
-      ansible.builtin.file:
-        path: "{{ item }}"
-        state: absent
-      loop:
-        - /home/{{ github_runner_user }}/runner
-      when: run_removal_tasks
-    - name: Add user '{{ github_runner_user }}' to group docker
-      user:
-        name: "{{ github_runner_user }}"
-        groups: docker
-        append: yes
-    - name: Restart docker service
-      ansible.builtin.service:
-        name: docker
-        state: restarted
-    - name: Install the runner [If error, RUNNER_CFG_PAT might be missing or expired! See README.md]
-      ansible.builtin.shell:
-        cmd: >
-          curl -s https://raw.githubusercontent.com/actions/runner/main/scripts/create-latest-svc.sh
-          | bash -s --
-          -s {{ github_scope }}
-          -n {{ github_runner_name }}
-          -l {{ github_runner_location }},self-hosted
-          -u {{ github_runner_user }}
-        chdir: /home/{{ github_runner_user }}
-      environment:
-        RUNNER_CFG_PAT: "{{ github_pat }}"
-      when: (not runner_dir.stat.exists) or (force_reinstall is defined and force_reinstall=="yes")
+    # Ansible task to automate:
+    # helm install "${INSTALLATION_NAME}" \
+    # --namespace "${NAMESPACE}" \
+    # --create-namespace \
+    # --set githubConfigUrl="https://github.com/caktus/philly-hip" \
+    # --set githubConfigSecret.github_token="${RUNNER_CFG_PAT}" \
+    # oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
+    - name: Configuring a runner scale set
+      kubernetes.core.helm:
+        context: "{{ k8s_context|mandatory }}"
+        chart_ref: oci://ghcr.io/actions/actions-runner-controller-charts/gha-runner-scale-set
+        chart_version: "{{ chart_version }}"
+        release_name: arc-runner-set
+        release_namespace: "{{ runner_namespace }}"
+        create_namespace: true
+        release_values:
+          githubConfigUrl: "https://github.com/caktus/philly-hip"
+          githubConfigSecret:
+            github_token: "{{ lookup('env', 'RUNNER_CFG_PAT') }}"
+        wait: yes

deploy/group_vars/all.yml

@@ -58,11 +58,6 @@ cloudformation_stack:
 
   template_parameters:
     UseAES256Encryption: "true"
-    AdministratorIPAddress: "{{ administrator_ip_cidrs[0] }}"  # Stack allows only single IP here to SSH to bastion
-    BastionAMI: "ami-0ad554caf874569d2" # https://cloud-images.ubuntu.com/locator/ec2/ [us-east-1 amd64]
-    BastionKeyName: rluna_hip
-    BastionInstanceType: t3.small # Is this a proper size?
-    BastionType: SSH
     CustomerManagedCmkArn: ""
     DomainName: "{{ app_name }}-prod.caktus-built.com"
     DomainNameAlternates: ""