feat(ci): use AKS overlaybd (#55)

Signed-off-by: Aviral Takkar <avtakkar@microsoft.com>
Azure · Jul 24, 2024 · e621e7c · e621e7c
1 parent 0bf05b7
commit e621e7c
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 51 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -60,8 +60,9 @@ jobs:
       id-token: write # This is required for requesting the JWT from AAD.
     env:
       TAG: ${{ needs.publish.outputs.git_tag }}
+      SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
     steps:
-      - name: 'Az CLI login'
+      - name: "Az CLI login"
         uses: azure/login@v1
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
@@ -86,7 +87,7 @@ jobs:
         with:
           ref: ${{ env.TAG }}
 
-      - name: 'Make'
+      - name: "Make"
         if: ${{ success() }}
         run: |
           PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-ctr
@@ -100,8 +101,9 @@ jobs:
       id-token: write # This is required for requesting the JWT from AAD.
     env:
       TAG: ${{ needs.publish.outputs.git_tag }}
+      SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
     steps:
-      - name: 'Az CLI login'
+      - name: "Az CLI login"
         uses: azure/login@v1
         with:
           client-id: ${{ secrets.AZURE_CLIENT_ID }}
@@ -125,8 +127,8 @@ jobs:
         uses: actions/checkout@v2
         with:
           ref: ${{ env.TAG }}
-  
-      - name: 'Make'
+
+      - name: "Make"
         if: ${{ success() }}
         run: |
           PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-streaming

diff --git a/build/ci/k8s/scanner.yml b/build/ci/k8s/scanner.yml
@@ -16,6 +16,13 @@ spec:
       labels:
         app: peerd-test
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: p2p-nodepool
+                operator: Exists
       initContainers:
         - name: sleep
           image: busybox

diff --git a/build/ci/k8s/teleport.yml b/build/ci/k8s/teleport.yml
@@ -11,45 +11,14 @@ data:
     #!/usr/bin/env bash
     set -xe
 
-    sudo apt-get update
-
-    pushd /tmp || exit 1
-
-    wget 'https://acrstreamingpackage.blob.core.windows.net/bin/latest/acr-mirror-2204.deb'
-    sudo apt-get install './acr-mirror-2204.deb' -y --allow-downgrades
-    rm './acr-mirror-2204.deb'
-    popd
-
-    systemctl enable --now acr-mirror
-
-    # These libraries are needed for overlaybd-tcmu
-    sudo /opt/acr/tools/overlaybd/install.sh
-    sudo /opt/acr/tools/overlaybd/enable-http-auth.sh
-    modprobe target_core_user
-
     # Enable overlaybd peer-to-peer
-    jq '.p2pConfig.enable = true' /etc/overlaybd/overlaybd.json > tmp.json
-    sudo mv tmp.json /etc/overlaybd/overlaybd.json
-
-    jq '.p2pConfig.address = "localhost:30000/blobs"' /etc/overlaybd/overlaybd.json > tmp.json
-    sudo mv tmp.json /etc/overlaybd/overlaybd.json
-
-    jq '.logConfig.logLevel = 0' /etc/overlaybd/overlaybd.json > tmp.json
-    sudo mv tmp.json /etc/overlaybd/overlaybd.json
-
-    cat /etc/overlaybd/overlaybd.json
-
-    # Enabling mirror for teleportmetricstest
-    curl -X PUT 'localhost:8578/config?ns=_default&enable_suffix=azurecr.io&stream_format=overlaybd&enable_containerd=true'
-
-    # Enable overlaybd
-    sudo /opt/acr/tools/overlaybd/enable.sh
-    touch /opt/sentinel
+    /opt/acr/tools/overlaybd/config.sh p2pConfig.enable true
+    /opt/acr/tools/overlaybd/config.sh p2pConfig.address \"http://localhost:30000/blobs\"  
+    /opt/acr/tools/overlaybd/config.sh logConfig.logLevel 0
 
-    # Install nerdctl: https://github.com/containerd/nerdctl/releases
-    wget https://github.com/containerd/nerdctl/releases/download/v1.6.0/nerdctl-1.6.0-linux-amd64.tar.gz
-    tar Cxzvvf /usr/local/bin nerdctl-1.6.0-linux-amd64.tar.gz
-    rm nerdctl-1.6.0-linux-amd64.tar.gz
+    # Restart overlaybd
+    sudo systemctl restart overlaybd-tcmu
+    sudo systemctl restart overlaybd-snapshotter
 ---
 apiVersion: apps/v1
 kind: DaemonSet
@@ -67,6 +36,13 @@ spec:
       labels:
         app: *name
     spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+              - matchExpressions:
+                  - key: p2p-nodepool
+                    operator: Exists
       hostNetwork: true
       hostPID: true
       containers:

diff --git a/build/ci/scripts/azure.sh b/build/ci/scripts/azure.sh
@@ -74,7 +74,7 @@ nodepool_deploy() {
     if [ "$DRY_RUN" == "false" ]; then
         echo "creating nodepool '$nodepool' in aks cluster '$aksName' in resource group '$rg'" && \
             az aks nodepool add --cluster-name $aksName --name $nodepool --resource-group $rg \
-                --mode User --labels "p2p-nodepool=$nodepool" --node-count 3 --node-vm-size Standard_D2s_v3
+                --mode User --labels "p2p-nodepool=true" --node-count 3 --node-vm-size Standard_D2s_v3  --enable-artifact-streaming
     else
         echo "[dry run] would have deployed nodepool '$nodepool' to aks cluster '$aksName' in resource group '$rg'"
     fi
@@ -85,8 +85,6 @@ peerd_helm_deploy() {
     local nodepool=$1
     local peerd_image_tag=$2
     local configureMirrors=$3
-
-    ensure_azure_token
 
     echo "deploying peerd to k8s cluster, chart: '$PEERD_HELM_CHART', tag: '$peerd_image_tag'" && \
         kubectl cluster-info
@@ -186,6 +184,8 @@ cmd__nodepool__up () {
     local peerd_image_tag=$PEERD_IMAGE_TAG
     local configureMirrors=$PEERD_CONFIGURE_MIRRORS
 
+    ensure_azure_token
+
     echo "get AKS credentials"
     get_aks_credentials $AKS_NAME $RESOURCE_GROUP
 
@@ -240,15 +240,18 @@ cmd__test__streaming() {
     if [ "$DRY_RUN" == "true" ]; then
         echo "[dry run] would have run test 'streaming'"
     else
+        echo "waiting 5 minutes" 
+        sleep 300
+
         echo "deploying acr mirror"
         kubectl apply -f $TELEPORT_DEPLOY_TEMPLATE
 
-        echo "waiting 5 minutes" 
-        sleep 300
+        echo "waiting 10 seconds" 
+        sleep 10
 
-        echo "deploying scanner app and waiting 2 minutes"
+        echo "deploying scanner app and waiting 1 minute"
         envsubst < $SCANNER_APP_DEPLOY_TEMPLATE | kubectl apply -f -
-        sleep 120
+        sleep 60
 
         echo "scanner logs"
         kubectl -n peerd-ns logs -l app=tests-scanner

diff --git a/build/ci/scripts/env.az.sh b/build/ci/scripts/env.az.sh
@@ -1,9 +1,12 @@
 #!/bin/bash
 set -e
 
-SUBSCRIPTION=""
-
 ensure_azure_token() {
+    if [ -z "$SUBSCRIPTION" ]; then
+        echo "Error: SUBSCRIPTION is not set."
+        exit 1
+    fi
+
     az account set --subscription $SUBSCRIPTION
 }