Cache e2e image builds

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>
rancher · Oct 22, 2024 · ddb1317 · ddb1317
1 parent c7700ae
commit ddb1317
Show file tree

Hide file tree

Showing 8 changed files with 96 additions and 22 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -39,3 +39,6 @@ cmd/clusterctl/clusterctl/**
 
 # Ignore go.work files
 go.work*
+
+# Ignore the buildx cache directory
+.buildx-cache/
diff --git a/.github/workflows/e2e-long-test.yaml b/.github/workflows/e2e-long-test.yaml
@@ -42,8 +42,15 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: '=1.22.0'
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
     - name: Run e2e tests
-      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/import-gitops make test-e2e
+      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/import-gitops CACHE_DIR=/tmp/.buildx-cache make test-e2e
       env:
         GINKGO_LABEL_FILTER: ${{ inputs.labels }}
     - name: Collect run artifacts
@@ -75,8 +82,15 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: '=1.22.0'
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
     - name: Run e2e tests
-      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/v2prov make test-e2e
+      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/v2prov CACHE_DIR=/tmp/.buildx-cache make test-e2e
       env:
         GINKGO_LABEL_FILTER: ${{ inputs.labels }}
     - name: Collect run artifacts
@@ -108,8 +122,15 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: '=1.22.0'
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
     - name: Run e2e tests
-      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/update-labels make test-e2e
+      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/update-labels CACHE_DIR=/tmp/.buildx-cache make test-e2e
       env:
         GINKGO_LABEL_FILTER: ${{ inputs.labels }}
     - name: Collect run artifacts
@@ -141,8 +162,15 @@ jobs:
       uses: actions/setup-go@v5
       with:
         go-version: '=1.22.0'
+    - name: Cache Docker layers
+      uses: actions/cache@v3
+      with:
+        path: /tmp/.buildx-cache
+        key: ${{ runner.os }}-buildx-${{ github.sha }}
+        restore-keys: |
+          ${{ runner.os }}-buildx-
     - name: Run e2e tests
-      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/embedded-capi-disabled make test-e2e
+      run: GINKGO_TESTS=$(pwd)/test/e2e/suites/embedded-capi-disabled CACHE_DIR=/tmp/.buildx-cache make test-e2e
       env:
         GINKGO_LABEL_FILTER: ${{ inputs.labels }}
     - name: Collect run artifacts

diff --git a/.github/workflows/e2e-short-test.yaml b/.github/workflows/e2e-short-test.yaml
@@ -20,8 +20,15 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "=1.22.0"
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
       - name: Run e2e tests
-        run: make test-e2e
+        run: CACHE_DIR=/tmp/.buildx-cache make test-e2e
       - name: Collect run artifacts
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/e2e-short.yaml b/.github/workflows/e2e-short.yaml
@@ -37,8 +37,15 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: "=1.22.0"
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
       - name: Run e2e tests
-        run: make test-e2e
+        run: CACHE_DIR=/tmp/.buildx-cache make test-e2e
       - name: Collect run artifacts
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/run-e2e-suite.yaml b/.github/workflows/run-e2e-suite.yaml
@@ -88,11 +88,15 @@ jobs:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: build and push e2e image
-        if: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT == 'eks' }}
-        run: make e2e-image-push
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
       - name: Run e2e tests
-        run: make test-e2e
+        run: CACHE_DIR=/tmp/.buildx-cache make test-e2e-push-image
       - name: Collect run artifacts
         if: always()
         uses: actions/upload-artifact@v4

diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml
@@ -20,6 +20,12 @@ jobs:
     name: Scan
     runs-on: "ubuntu-20.04"
     steps:
+      - name: Docker login
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Checkout code
         uses: actions/checkout@v4.1.7
       - name: Build an image

diff --git a/.gitignore b/.gitignore
@@ -37,4 +37,6 @@ hack/crd/bases/*
 .secrets
 
 # helm
-**/Chart.lock
+**/Chart.lock
+
+.buildx-cache/
diff --git a/Makefile b/Makefile
@@ -196,6 +196,8 @@ PULL_POLICY ?= IfNotPresent
 # Development config
 RANCHER_HOSTNAME ?= my.hostname.dev
 
+CACHE_DIR ?= .buildx-cache/
+CACHE_COMMANDS = "--cache-from type=local,src=$(CACHE_DIR) --cache-to type=local,dest=$(CACHE_DIR),mode=max"
 
 .PHONY: all
 all: build
@@ -349,7 +351,7 @@ docker-pull-prerequisites:
 docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
 ## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368)
 	# buildx does not support using local registry for multi-architecture images
-	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
 			--platform $(ARCH) \
 			--load \
 			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
@@ -359,7 +361,7 @@ docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build dock
 
 .PHONY: docker-build-and-push-etcdrestore
 docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures
-	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+	cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
 			--platform $(TARGET_PLATFORMS) \
 			--push \
 			--sbom=true \
@@ -373,7 +375,7 @@ docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## R
 .PHONY: docker-build
 docker-build: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture
 	# buildx does not support using local registry for multi-architecture images
-	DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+	DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
 			--platform $(ARCH) \
 			--load \
 			--build-arg builder_image=$(GO_CONTAINER_IMAGE) \
@@ -383,7 +385,7 @@ docker-build: buildx-machine docker-pull-prerequisites ## Build docker image for
 
 .PHONY: docker-build-and-push
 docker-build-and-push: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push targets for all architectures
-	DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \
+	DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \
 			--platform $(TARGET_PLATFORMS) \
 			--push \
 			--sbom=true \
@@ -567,22 +569,37 @@ release-chart: $(HELM) $(NOTES) build-chart verify-gen
 	$(NOTES) --repository $(REPO) -workers=1 -add-kubernetes-version-support=false --from=$(PREVIOUS_TAG) > $(CHART_RELEASE_DIR)/RELEASE_NOTES.md
 	$(HELM) package $(CHART_RELEASE_DIR) --app-version=$(HELM_CHART_TAG) --version=$(HELM_CHART_TAG) --destination=$(CHART_PACKAGE_DIR)
 
-.PHONY: test-e2e
-test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests
-	$(E2ECONFIG_VARS) $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
+
+## --------------------------------------
+## E2E Tests
+## --------------------------------------
+
+$(CACHE_DIR):
+	mkdir -p $(CACHE_DIR)/
+
+E2E_RUN_COMMAND=$(E2ECONFIG_VARS) $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \
 		-poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \
 		$(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \
 		--output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(GINKGO_TESTS) -- \
 	    -e2e.config="$(E2E_CONF_FILE)"
 
+.PHONY: test-e2e
+test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests
+	$(E2E_RUN_COMMAND)
+
+.PHONY: test-e2e-push-image
+test-e2e-push-image: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image-push
+	$(E2E_RUN_COMMAND)
+
 .PHONY: e2e-image
-e2e-image: ## Build the image for e2e tests
-	TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build
+e2e-image: $(CACHE_DIR) ## Build the image for e2e tests
+	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build
 	RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e CONTROLLER_IMAGE_VERSION=v0.0.1 $(MAKE) build-chart
 
 .PHONY: e2e-image-push
-e2e-image-push: e2e-image ## Push the image for e2e tests
-	TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build-and-push
+e2e-image-push: $(CACHE_DIR) ## Push the image for e2e tests
+	TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e \
+	ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push
 
 .PHONY: compile-e2e
 e2e-compile: ## Test e2e compilation