From e621e7cb18498284e4f4528fbc2fa824639f7d2c Mon Sep 17 00:00:00 2001 From: Aviral Takkar <39969667+avtakkar@users.noreply.github.com> Date: Tue, 23 Jul 2024 16:56:12 -0700 Subject: [PATCH] feat(ci): use AKS overlaybd (#55) Signed-off-by: Aviral Takkar --- .github/workflows/release.yml | 12 +++++---- build/ci/k8s/scanner.yml | 7 +++++ build/ci/k8s/teleport.yml | 50 +++++++++-------------------------- build/ci/scripts/azure.sh | 17 +++++++----- build/ci/scripts/env.az.sh | 7 +++-- 5 files changed, 42 insertions(+), 51 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 16505f2..7d34fe4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -60,8 +60,9 @@ jobs: id-token: write # This is required for requesting the JWT from AAD. env: TAG: ${{ needs.publish.outputs.git_tag }} + SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION_ID }} steps: - - name: 'Az CLI login' + - name: "Az CLI login" uses: azure/login@v1 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} @@ -86,7 +87,7 @@ jobs: with: ref: ${{ env.TAG }} - - name: 'Make' + - name: "Make" if: ${{ success() }} run: | PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-ctr @@ -100,8 +101,9 @@ jobs: id-token: write # This is required for requesting the JWT from AAD. env: TAG: ${{ needs.publish.outputs.git_tag }} + SUBSCRIPTION: ${{ secrets.AZURE_SUBSCRIPTION_ID }} steps: - - name: 'Az CLI login' + - name: "Az CLI login" uses: azure/login@v1 with: client-id: ${{ secrets.AZURE_CLIENT_ID }} @@ -125,8 +127,8 @@ jobs: uses: actions/checkout@v2 with: ref: ${{ env.TAG }} - - - name: 'Make' + + - name: "Make" if: ${{ success() }} run: | PEERD_IMAGE_TAG=${{ env.TAG }} make tests-deps-install ci-aks-streaming diff --git a/build/ci/k8s/scanner.yml b/build/ci/k8s/scanner.yml index 1696529..0335052 100644 --- a/build/ci/k8s/scanner.yml +++ b/build/ci/k8s/scanner.yml @@ -16,6 +16,13 @@ spec: labels: app: peerd-test spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: p2p-nodepool + operator: Exists initContainers: - name: sleep image: busybox diff --git a/build/ci/k8s/teleport.yml b/build/ci/k8s/teleport.yml index 36722b5..b34c0c6 100644 --- a/build/ci/k8s/teleport.yml +++ b/build/ci/k8s/teleport.yml @@ -11,45 +11,14 @@ data: #!/usr/bin/env bash set -xe - sudo apt-get update - - pushd /tmp || exit 1 - - wget 'https://acrstreamingpackage.blob.core.windows.net/bin/latest/acr-mirror-2204.deb' - sudo apt-get install './acr-mirror-2204.deb' -y --allow-downgrades - rm './acr-mirror-2204.deb' - popd - - systemctl enable --now acr-mirror - - # These libraries are needed for overlaybd-tcmu - sudo /opt/acr/tools/overlaybd/install.sh - sudo /opt/acr/tools/overlaybd/enable-http-auth.sh - modprobe target_core_user - # Enable overlaybd peer-to-peer - jq '.p2pConfig.enable = true' /etc/overlaybd/overlaybd.json > tmp.json - sudo mv tmp.json /etc/overlaybd/overlaybd.json - - jq '.p2pConfig.address = "localhost:30000/blobs"' /etc/overlaybd/overlaybd.json > tmp.json - sudo mv tmp.json /etc/overlaybd/overlaybd.json - - jq '.logConfig.logLevel = 0' /etc/overlaybd/overlaybd.json > tmp.json - sudo mv tmp.json /etc/overlaybd/overlaybd.json - - cat /etc/overlaybd/overlaybd.json - - # Enabling mirror for teleportmetricstest - curl -X PUT 'localhost:8578/config?ns=_default&enable_suffix=azurecr.io&stream_format=overlaybd&enable_containerd=true' - - # Enable overlaybd - sudo /opt/acr/tools/overlaybd/enable.sh - touch /opt/sentinel + /opt/acr/tools/overlaybd/config.sh p2pConfig.enable true + /opt/acr/tools/overlaybd/config.sh p2pConfig.address \"http://localhost:30000/blobs\" + /opt/acr/tools/overlaybd/config.sh logConfig.logLevel 0 - # Install nerdctl: https://github.com/containerd/nerdctl/releases - wget https://github.com/containerd/nerdctl/releases/download/v1.6.0/nerdctl-1.6.0-linux-amd64.tar.gz - tar Cxzvvf /usr/local/bin nerdctl-1.6.0-linux-amd64.tar.gz - rm nerdctl-1.6.0-linux-amd64.tar.gz + # Restart overlaybd + sudo systemctl restart overlaybd-tcmu + sudo systemctl restart overlaybd-snapshotter --- apiVersion: apps/v1 kind: DaemonSet @@ -67,6 +36,13 @@ spec: labels: app: *name spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: p2p-nodepool + operator: Exists hostNetwork: true hostPID: true containers: diff --git a/build/ci/scripts/azure.sh b/build/ci/scripts/azure.sh index b813100..e903d54 100755 --- a/build/ci/scripts/azure.sh +++ b/build/ci/scripts/azure.sh @@ -74,7 +74,7 @@ nodepool_deploy() { if [ "$DRY_RUN" == "false" ]; then echo "creating nodepool '$nodepool' in aks cluster '$aksName' in resource group '$rg'" && \ az aks nodepool add --cluster-name $aksName --name $nodepool --resource-group $rg \ - --mode User --labels "p2p-nodepool=$nodepool" --node-count 3 --node-vm-size Standard_D2s_v3 + --mode User --labels "p2p-nodepool=true" --node-count 3 --node-vm-size Standard_D2s_v3 --enable-artifact-streaming else echo "[dry run] would have deployed nodepool '$nodepool' to aks cluster '$aksName' in resource group '$rg'" fi @@ -85,8 +85,6 @@ peerd_helm_deploy() { local nodepool=$1 local peerd_image_tag=$2 local configureMirrors=$3 - - ensure_azure_token echo "deploying peerd to k8s cluster, chart: '$PEERD_HELM_CHART', tag: '$peerd_image_tag'" && \ kubectl cluster-info @@ -186,6 +184,8 @@ cmd__nodepool__up () { local peerd_image_tag=$PEERD_IMAGE_TAG local configureMirrors=$PEERD_CONFIGURE_MIRRORS + ensure_azure_token + echo "get AKS credentials" get_aks_credentials $AKS_NAME $RESOURCE_GROUP @@ -240,15 +240,18 @@ cmd__test__streaming() { if [ "$DRY_RUN" == "true" ]; then echo "[dry run] would have run test 'streaming'" else + echo "waiting 5 minutes" + sleep 300 + echo "deploying acr mirror" kubectl apply -f $TELEPORT_DEPLOY_TEMPLATE - echo "waiting 5 minutes" - sleep 300 + echo "waiting 10 seconds" + sleep 10 - echo "deploying scanner app and waiting 2 minutes" + echo "deploying scanner app and waiting 1 minute" envsubst < $SCANNER_APP_DEPLOY_TEMPLATE | kubectl apply -f - - sleep 120 + sleep 60 echo "scanner logs" kubectl -n peerd-ns logs -l app=tests-scanner diff --git a/build/ci/scripts/env.az.sh b/build/ci/scripts/env.az.sh index f52f22a..b05dcc2 100755 --- a/build/ci/scripts/env.az.sh +++ b/build/ci/scripts/env.az.sh @@ -1,9 +1,12 @@ #!/bin/bash set -e -SUBSCRIPTION="" - ensure_azure_token() { + if [ -z "$SUBSCRIPTION" ]; then + echo "Error: SUBSCRIPTION is not set." + exit 1 + fi + az account set --subscription $SUBSCRIPTION }