feat: add streaming ci (#24)

Signed-off-by: Aviral Takkar <avtakkar@microsoft.com>
Azure · Jul 24, 2024 · e6b29ee · e6b29ee
1 parent fe9387b
commit e6b29ee
Show file tree

Hide file tree

Showing 9 changed files with 273 additions and 21 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -51,8 +51,8 @@ jobs:
         run: |
           docker push ${{ env.REGISTRY }}/${{ env.REPO_PREFIX }}peerd:${{ steps.get_image_tag.outputs.docker_tag }}
 
-  ci:
-    name: Run AKS CI
+  ciCtr:
+    name: Run AKS CTR CI
     runs-on: ubuntu-latest
     needs: publish
     permissions:
@@ -78,3 +78,32 @@ jobs:
         if: ${{ success() }}
         run: |
           PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-ctr
+
+  ciStreaming:
+    name: Run AKS Streaming CI
+    runs-on: ubuntu-latest
+    needs: [publish, ciCtr]
+    permissions:
+      contents: read
+      id-token: write # This is required for requesting the JWT from AAD.
+    env:
+      TAG: ${{ needs.publish.outputs.git_tag }}
+    steps:
+      - name: 'Az CLI login'
+        uses: azure/login@v1
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+
+      - name: Check Out Source Code
+        if: ${{ success() }}
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ env.TAG }}
+
+      - name: 'Make'
+        if: ${{ success() }}
+        run: |
+          PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-streaming
+
diff --git a/.gitignore b/.gitignore
@@ -22,4 +22,7 @@ bin/**/*
 go.work
 
 # Directories mounted to CI cluster.
-build/ci/configs/certs.d
+build/ci/configs/certs.d
+
+# CI generated files.
+.kraterdev/
diff --git a/build/ci/Makefile b/build/ci/Makefile
@@ -49,12 +49,26 @@ ci-kind-random: build-image tests-random-image kind-delete kind-create kind-depl
 ci-aks-ctr: ## Run the containerd hosts CI in AKS.
 	@echo "+ $@"
 ifndef NODEPOOL
-	$(eval NODEPOOL := $(shell date +"p2p%y%m%d"))
+	$(eval NODEPOOL := $(shell date +"prel%y%m%d"))
 endif
 ifndef PEERD_IMAGE_TAG
 	$(eval PEERD_IMAGE_TAG := "dev")
 endif
 	@echo "\033[92mRunning CI NODEPOOL: $(NODEPOOL)\033[0m"
-	@( PEERD_IMAGE_TAG=$(PEERD_IMAGE_TAG) $(SCRIPTS_DIR)/azure.sh nodepool up -y $(NODEPOOL) )
+	@( PEERD_IMAGE_TAG=$(PEERD_IMAGE_TAG) PEERD_CONFIGURE_MIRRORS=true $(SCRIPTS_DIR)/azure.sh nodepool up -y $(NODEPOOL) )
 	@( $(SCRIPTS_DIR)/azure.sh test ctr -y $(NODEPOOL) )
 	@( $(SCRIPTS_DIR)/azure.sh nodepool delete -y $(NODEPOOL) )
+
+.PHONY: ci-aks-streaming
+ci-aks-streaming: ## Run the streaming CI in AKS.
+	@echo "+ $@"
+ifndef NODEPOOL
+	$(eval NODEPOOL := $(shell date +"prels%y%m%d"))
+endif
+ifndef PEERD_IMAGE_TAG
+	$(eval PEERD_IMAGE_TAG := "dev")
+endif
+	@echo "\033[92mRunning CI NODEPOOL: $(NODEPOOL)\033[0m"
+	@( PEERD_IMAGE_TAG=$(PEERD_IMAGE_TAG) PEERD_CONFIGURE_MIRRORS=false $(SCRIPTS_DIR)/azure.sh nodepool up -y $(NODEPOOL) )
+	@( $(SCRIPTS_DIR)/azure.sh test streaming -y $(NODEPOOL) )
+	@( $(SCRIPTS_DIR)/azure.sh nodepool delete -y $(NODEPOOL) )
diff --git a/build/ci/k8s/scanner.yml b/build/ci/k8s/scanner.yml
@@ -0,0 +1,63 @@
+# Runs the 'scanner' benchmark scenario on all cluster nodes.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: tests-scanner-actions
+  namespace: peerd-ns
+  labels:
+    app: tests-scanner
+data:
+  wasm: |
+    #!/usr/bin/env bash
+    set -xe
+
+    nerdctl run \
+      --hosts-dir "/etc/containerd/certs.d" \
+      --snapshotter=overlaybd \
+      --net host -i --rm $TESTS_SCANNER_IMAGE
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: &name tests-scanner
+  namespace: peerd-ns
+  labels:
+    app: *name
+spec:
+  selector:
+    matchLabels:
+      app: *name
+  template:
+    metadata:
+      labels:
+        app: *name
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+      - image: docker.io/alexeldeib/nsenter:latest # https://github.com/alexeldeib/azbench/blob/main/images/nsenter/entrypoint.sh
+        imagePullPolicy: Always
+        name: *name
+        args: ["wasm"]
+        resources:
+          # requests:
+          #   cpu: 0.5
+          #   memory: 2000Mi
+          # limits:
+          #   cpu: 0.5
+          #   memory: 2000Mi
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: actions
+          mountPath: "/opt/actions"
+        - name: hostmount
+          mountPath: "/mnt/actions"
+      volumes:
+      - name: hostmount
+        hostPath:
+          path: /opt/actions
+          type: DirectoryOrCreate
+      - name: actions
+        configMap:
+          name: tests-scanner-actions
diff --git a/build/ci/k8s/teleport.yml b/build/ci/k8s/teleport.yml
@@ -0,0 +1,98 @@
+# Installs ACR mirror.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: teleport-actions
+  namespace: peerd-ns
+  labels:
+    app: teleport
+data:
+  wasm: |
+    #!/usr/bin/env bash
+    set -xe
+
+    sudo apt-get update
+
+    pushd /tmp || exit 1
+
+    wget 'https://acrstreamingpackage.blob.core.windows.net/bin/latest/acr-mirror-2204.deb'
+    sudo apt-get install './acr-mirror-2204.deb' -y --allow-downgrades
+    rm './acr-mirror-2204.deb'
+    popd
+
+    systemctl enable --now acr-mirror
+
+    # These libraries are needed for overlaybd-tcmu
+    sudo /opt/acr/tools/overlaybd/install.sh
+    sudo /opt/acr/tools/overlaybd/enable-http-auth.sh
+    modprobe target_core_user
+
+    # Enable overlaybd peer-to-peer
+    jq '.p2pConfig.enable = true' /etc/overlaybd/overlaybd.json > tmp.json
+    sudo mv tmp.json /etc/overlaybd/overlaybd.json
+
+    jq '.p2pConfig.address = "localhost:30000/blobs"' /etc/overlaybd/overlaybd.json > tmp.json
+    sudo mv tmp.json /etc/overlaybd/overlaybd.json
+
+    jq '.logConfig.logLevel = 0' /etc/overlaybd/overlaybd.json > tmp.json
+    sudo mv tmp.json /etc/overlaybd/overlaybd.json
+
+    cat /etc/overlaybd/overlaybd.json
+
+    # Enabling mirror for teleportmetricstest
+    curl -X PUT 'localhost:8578/config?ns=_default&enable_suffix=azurecr.io&stream_format=overlaybd&enable_containerd=true'
+
+    # Enable overlaybd
+    sudo /opt/acr/tools/overlaybd/enable.sh
+    touch /opt/sentinel
+
+    # Install nerdctl: https://github.com/containerd/nerdctl/releases
+    wget https://github.com/containerd/nerdctl/releases/download/v1.6.0/nerdctl-1.6.0-linux-amd64.tar.gz
+    tar Cxzvvf /usr/local/bin nerdctl-1.6.0-linux-amd64.tar.gz
+    rm nerdctl-1.6.0-linux-amd64.tar.gz
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: &name teleport
+  namespace: peerd-ns
+  labels:
+    app: *name
+spec:
+  selector:
+    matchLabels:
+      app: *name
+  template:
+    metadata:
+      labels:
+        app: *name
+    spec:
+      hostNetwork: true
+      hostPID: true
+      containers:
+        - image: docker.io/alexeldeib/nsenter:latest # https://github.com/alexeldeib/azbench/blob/main/images/nsenter/entrypoint.sh
+          imagePullPolicy: Always
+          name: *name
+          args: ["wasm"]
+          resources:
+            requests:
+              cpu: 0.5
+              memory: 2000Mi
+            limits:
+              cpu: 0.5
+              memory: 2000Mi
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: actions
+              mountPath: "/opt/actions"
+            - name: hostmount
+              mountPath: "/mnt/actions"
+      volumes:
+        - name: hostmount
+          hostPath:
+            path: /opt/actions
+            type: DirectoryOrCreate
+        - name: actions
+          configMap:
+            name: teleport-actions
diff --git a/build/ci/scripts/azure.sh b/build/ci/scripts/azure.sh
@@ -4,8 +4,10 @@ set -e
 SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
 source $SCRIPT_DIR/env.sh
 
-PEERD_HELM_CHART="$SCRIPT_DIR/../build/package/peerd-helm"
-TESTS_AZURE_CLI_DEPLOY_TEMPLATE=$SCRIPT_DIR/../build/ci/k8s/azure-cli.yml
+PEERD_HELM_CHART="$SCRIPT_DIR/../../package/peerd-helm"
+TELEPORT_DEPLOY_TEMPLATE="$SCRIPT_DIR/../k8s/teleport.yml"
+SCANNER_APP_DEPLOY_TEMPLATE="$SCRIPT_DIR/../k8s/scanner.yml"
+TESTS_AZURE_CLI_DEPLOY_TEMPLATE=$SCRIPT_DIR/../k8s/azure-cli.yml
 
 show_help() {
     usageStr="
@@ -45,6 +47,12 @@ Sub commands:
 
 * confirm: run the ctr test on 'nodepool1'
     $(basename $0) test ctr -y 'nodepool1'
+
+* dry run: runs the streaming test on 'nodepool1'
+    $(basename $0) test streaming 'nodepool1'
+
+* confirm: run the streaming test on 'nodepool1'
+    $(basename $0) test streaming -y 'nodepool1'
 "
     echo "$usageStr"
 }
@@ -76,6 +84,7 @@ nodepool_deploy() {
 peerd_helm_deploy() {
     local nodepool=$1
     local peerd_image_tag=$2
+    local configureMirrors=$3
 
     ensure_azure_token
 
@@ -85,7 +94,8 @@ peerd_helm_deploy() {
     if [ "$DRY_RUN" == "false" ]; then
         HELM_RELEASE_NAME=peerd && \
             helm install --wait $HELM_RELEASE_NAME $PEERD_HELM_CHART \
-                --set "peerd.image.ref=ghcr.io/azure/acr/dev/peerd:$peerd_image_tag"
+                --set "peerd.image.ref=ghcr.io/azure/acr/dev/peerd:$peerd_image_tag" \
+                --set "peerd.configureMirrors=$configureMirrors"
     else
         echo "[dry run] would have deployed app to k8s cluster"
     fi
@@ -173,6 +183,7 @@ cmd__nodepool__delete() {
 cmd__nodepool__up () {
     local nodepool=$1
     local peerd_image_tag=$PEERD_IMAGE_TAG
+    local configureMirrors=$PEERD_CONFIGURE_MIRRORS
 
     echo "get AKS credentials"
     get_aks_credentials $AKS_NAME $RESOURCE_GROUP
@@ -184,7 +195,7 @@ cmd__nodepool__up () {
     nodepool_deploy $AKS_NAME $RESOURCE_GROUP $nodepool
 
     echo "deploying peerd helm chart using tag '$peerd_image_tag'"
-    peerd_helm_deploy $nodepool $peerd_image_tag
+    peerd_helm_deploy $nodepool $peerd_image_tag $configureMirrors
 
     echo "waiting for pods to connect"
     wait_for_peerd_pods $AKS_NAME $RESOURCE_GROUP $nodepool "P2PConnected"
@@ -200,23 +211,61 @@ cmd__test__ctr() {
     if [ "$DRY_RUN" == "true" ]; then
         echo "[dry run] would have run test 'ctr'"
     else
-        # Get nodes
-        nodes=$(kubectl get nodes -o jsonpath='{.items[*].metadata.name}')
-        echo "nodes: $nodes"
-        total=`echo "$nodes" | tr -s " " "\012" | wc -l`
-
         # Pull the image on all nodes and verify that at least one P2PActive event is generated.
         kubectl apply -f $TESTS_AZURE_CLI_DEPLOY_TEMPLATE
 
         wait_for_peerd_pods $context $AKS_NAME $RESOURCE_GROUP $nodepool "P2PActive" 1
 
         echo "fetching metrics from pods"
         print_peerd_metrics
+
+        echo "cleaning up apps"
+        helm uninstall peerd --ignore-not-found=true
+        kubectl delete -f $TESTS_AZURE_CLI_DEPLOY_TEMPLATE
+
+        echo "test 'ctr' complete"
     fi
 
     print_and_exit_if_dry_run
 }
 
+cmd__test__streaming() {
+    aksName=$AKS_NAME
+    rg=$RESOURCE_GROUP
+    local nodepool=$1
+
+    echo "running test 'streaming'"
+
+    if [ "$DRY_RUN" == "true" ]; then
+        echo "[dry run] would have run test 'streaming'"
+    else
+        echo "deploying acr mirror"
+        kubectl apply -f $TELEPORT_DEPLOY_TEMPLATE
+
+        echo "waiting 5 minutes" 
+        sleep 300
+
+        echo "deploying scanner app and waiting 2 minutes"
+        envsubst < $SCANNER_APP_DEPLOY_TEMPLATE | kubectl apply -f -
+        sleep 120
+
+        echo "scanner logs"
+        kubectl -n peerd-ns logs -l app=tests-scanner
+
+        wait_for_peerd_pods $context $AKS_NAME $RESOURCE_GROUP $nodepool "P2PActive" 1
+
+        echo "fetching metrics from pods"
+        print_peerd_metrics
+
+        echo "cleaning up apps"
+        helm uninstall peerd --ignore-not-found=true
+        kubectl delete -f $SCANNER_APP_DEPLOY_TEMPLATE
+
+        echo "test 'streaming' complete"
+    fi
+
+    print_and_exit_if_dry_run
+}
 
 # Initialize script.
 if [[ -z "$DRY_RUN" ]]; then

diff --git a/build/ci/scripts/env.sh b/build/ci/scripts/env.sh
@@ -9,6 +9,7 @@ RESOURCE_GROUP="p2p-ci-rg"
 LOCATION="westus2"
 AKS_NAME="acrp2pciaks"
 ACR_NAME="acrp2pci"
+export TESTS_SCANNER_IMAGE="acrp2pci.azurecr.io/scanner:latest"
 
 indent() {
     sed 's/^/  /'

diff --git a/build/package/peerd-helm/templates/app.yml b/build/package/peerd-helm/templates/app.yml
@@ -35,6 +35,7 @@ spec:
           args:
             - "--log-level=debug"
             - "run"
+            - "--http-addr=0.0.0.0:5000"
             - "--add-mirror-configuration={{ .Values.peerd.configureMirrors }}"
             {{- with .Values.peerd.hosts }}
             - --hosts
@@ -85,6 +86,7 @@ spec:
     - name: http
       protocol: TCP
       port: 5000
+      nodePort: 30000
       targetPort: http
     - name: https
       protocol: TCP