From ddb1317dce2e46f2f73face42a2ad9366e3ec3ff Mon Sep 17 00:00:00 2001 From: Alexandr Demicev Date: Mon, 21 Oct 2024 16:20:51 +0200 Subject: [PATCH] Cache e2e image builds Signed-off-by: Alexandr Demicev --- .dockerignore | 3 +++ .github/workflows/e2e-long-test.yaml | 36 ++++++++++++++++++++++--- .github/workflows/e2e-short-test.yaml | 9 ++++++- .github/workflows/e2e-short.yaml | 9 ++++++- .github/workflows/run-e2e-suite.yaml | 12 ++++++--- .github/workflows/trivy.yml | 6 +++++ .gitignore | 4 ++- Makefile | 39 +++++++++++++++++++-------- 8 files changed, 96 insertions(+), 22 deletions(-) diff --git a/.dockerignore b/.dockerignore index 2ceee2fda..30e1fac4f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -39,3 +39,6 @@ cmd/clusterctl/clusterctl/** # Ignore go.work files go.work* + +# Ignore the buildx cache directory +.buildx-cache/ diff --git a/.github/workflows/e2e-long-test.yaml b/.github/workflows/e2e-long-test.yaml index adf7c8a01..667343ec9 100644 --- a/.github/workflows/e2e-long-test.yaml +++ b/.github/workflows/e2e-long-test.yaml @@ -42,8 +42,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: '=1.22.0' + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: GINKGO_TESTS=$(pwd)/test/e2e/suites/import-gitops make test-e2e + run: GINKGO_TESTS=$(pwd)/test/e2e/suites/import-gitops CACHE_DIR=/tmp/.buildx-cache make test-e2e env: GINKGO_LABEL_FILTER: ${{ inputs.labels }} - name: Collect run artifacts @@ -75,8 +82,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: '=1.22.0' + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: GINKGO_TESTS=$(pwd)/test/e2e/suites/v2prov make test-e2e + run: GINKGO_TESTS=$(pwd)/test/e2e/suites/v2prov CACHE_DIR=/tmp/.buildx-cache make test-e2e env: GINKGO_LABEL_FILTER: ${{ inputs.labels }} - name: Collect run artifacts @@ -108,8 +122,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: '=1.22.0' + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: GINKGO_TESTS=$(pwd)/test/e2e/suites/update-labels make test-e2e + run: GINKGO_TESTS=$(pwd)/test/e2e/suites/update-labels CACHE_DIR=/tmp/.buildx-cache make test-e2e env: GINKGO_LABEL_FILTER: ${{ inputs.labels }} - name: Collect run artifacts @@ -141,8 +162,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: '=1.22.0' + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: GINKGO_TESTS=$(pwd)/test/e2e/suites/embedded-capi-disabled make test-e2e + run: GINKGO_TESTS=$(pwd)/test/e2e/suites/embedded-capi-disabled CACHE_DIR=/tmp/.buildx-cache make test-e2e env: GINKGO_LABEL_FILTER: ${{ inputs.labels }} - name: Collect run artifacts diff --git a/.github/workflows/e2e-short-test.yaml b/.github/workflows/e2e-short-test.yaml index 876e2510f..e6481690b 100644 --- a/.github/workflows/e2e-short-test.yaml +++ b/.github/workflows/e2e-short-test.yaml @@ -20,8 +20,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: "=1.22.0" + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: make test-e2e + run: CACHE_DIR=/tmp/.buildx-cache make test-e2e - name: Collect run artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/e2e-short.yaml b/.github/workflows/e2e-short.yaml index fa75eae90..746e40874 100644 --- a/.github/workflows/e2e-short.yaml +++ b/.github/workflows/e2e-short.yaml @@ -37,8 +37,15 @@ jobs: uses: actions/setup-go@v5 with: go-version: "=1.22.0" + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: make test-e2e + run: CACHE_DIR=/tmp/.buildx-cache make test-e2e - name: Collect run artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/run-e2e-suite.yaml b/.github/workflows/run-e2e-suite.yaml index fb9716f66..951694cea 100644 --- a/.github/workflows/run-e2e-suite.yaml +++ b/.github/workflows/run-e2e-suite.yaml @@ -88,11 +88,15 @@ jobs: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: build and push e2e image - if: ${{ inputs.MANAGEMENT_CLUSTER_ENVIRONMENT == 'eks' }} - run: make e2e-image-push + - name: Cache Docker layers + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Run e2e tests - run: make test-e2e + run: CACHE_DIR=/tmp/.buildx-cache make test-e2e-push-image - name: Collect run artifacts if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index f902ee796..744397bee 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -20,6 +20,12 @@ jobs: name: Scan runs-on: "ubuntu-20.04" steps: + - name: Docker login + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Checkout code uses: actions/checkout@v4.1.7 - name: Build an image diff --git a/.gitignore b/.gitignore index b3855caf8..820b135e5 100644 --- a/.gitignore +++ b/.gitignore @@ -37,4 +37,6 @@ hack/crd/bases/* .secrets # helm -**/Chart.lock \ No newline at end of file +**/Chart.lock + +.buildx-cache/ \ No newline at end of file diff --git a/Makefile b/Makefile index 1c942bcce..290d4a839 100644 --- a/Makefile +++ b/Makefile @@ -196,6 +196,8 @@ PULL_POLICY ?= IfNotPresent # Development config RANCHER_HOSTNAME ?= my.hostname.dev +CACHE_DIR ?= .buildx-cache/ +CACHE_COMMANDS = "--cache-from type=local,src=$(CACHE_DIR) --cache-to type=local,dest=$(CACHE_DIR),mode=max" .PHONY: all all: build @@ -349,7 +351,7 @@ docker-pull-prerequisites: docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture ## reads Dockerfile from stdin to avoid an incorrectly cached Dockerfile (https://github.com/moby/buildkit/issues/1368) # buildx does not support using local registry for multi-architecture images - cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \ + cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ --platform $(ARCH) \ --load \ --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ @@ -359,7 +361,7 @@ docker-build-etcdrestore: buildx-machine docker-pull-prerequisites ## Build dock .PHONY: docker-build-and-push-etcdrestore docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push-etcdrestore targets for all architectures - cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \ + cat $(EXP_ETCDRESTORE_DIR)/Dockerfile | DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ --platform $(TARGET_PLATFORMS) \ --push \ --sbom=true \ @@ -373,7 +375,7 @@ docker-build-and-push-etcdrestore: buildx-machine docker-pull-prerequisites ## R .PHONY: docker-build docker-build: buildx-machine docker-pull-prerequisites ## Build docker image for a specific architecture # buildx does not support using local registry for multi-architecture images - DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \ + DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ --platform $(ARCH) \ --load \ --build-arg builder_image=$(GO_CONTAINER_IMAGE) \ @@ -383,7 +385,7 @@ docker-build: buildx-machine docker-pull-prerequisites ## Build docker image for .PHONY: docker-build-and-push docker-build-and-push: buildx-machine docker-pull-prerequisites ## Run docker-build-and-push targets for all architectures - DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build \ + DOCKER_BUILDKIT=1 BUILDX_BUILDER=$(MACHINE) docker buildx build $(ADDITIONAL_COMMANDS) \ --platform $(TARGET_PLATFORMS) \ --push \ --sbom=true \ @@ -567,22 +569,37 @@ release-chart: $(HELM) $(NOTES) build-chart verify-gen $(NOTES) --repository $(REPO) -workers=1 -add-kubernetes-version-support=false --from=$(PREVIOUS_TAG) > $(CHART_RELEASE_DIR)/RELEASE_NOTES.md $(HELM) package $(CHART_RELEASE_DIR) --app-version=$(HELM_CHART_TAG) --version=$(HELM_CHART_TAG) --destination=$(CHART_PACKAGE_DIR) -.PHONY: test-e2e -test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests - $(E2ECONFIG_VARS) $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ + +## -------------------------------------- +## E2E Tests +## -------------------------------------- + +$(CACHE_DIR): + mkdir -p $(CACHE_DIR)/ + +E2E_RUN_COMMAND=$(E2ECONFIG_VARS) $(GINKGO) -v --trace -poll-progress-after=$(GINKGO_POLL_PROGRESS_AFTER) \ -poll-progress-interval=$(GINKGO_POLL_PROGRESS_INTERVAL) --tags=e2e --focus="$(GINKGO_FOCUS)" --label-filter="$(GINKGO_LABEL_FILTER)" \ $(_SKIP_ARGS) --nodes=$(GINKGO_NODES) --timeout=$(GINKGO_TIMEOUT) --no-color=$(GINKGO_NOCOLOR) \ --output-dir="$(ARTIFACTS)" --junit-report="junit.e2e_suite.1.xml" $(GINKGO_ARGS) $(GINKGO_TESTS) -- \ -e2e.config="$(E2E_CONF_FILE)" +.PHONY: test-e2e +test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-end tests + $(E2E_RUN_COMMAND) + +.PHONY: test-e2e-push-image +test-e2e-push-image: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image-push + $(E2E_RUN_COMMAND) + .PHONY: e2e-image -e2e-image: ## Build the image for e2e tests - TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build +e2e-image: $(CACHE_DIR) ## Build the image for e2e tests + ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build RELEASE_TAG=v0.0.1 CONTROLLER_IMG=$(REGISTRY)/$(ORG)/turtles-e2e CONTROLLER_IMAGE_VERSION=v0.0.1 $(MAKE) build-chart .PHONY: e2e-image-push -e2e-image-push: e2e-image ## Push the image for e2e tests - TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e $(MAKE) docker-build-and-push +e2e-image-push: $(CACHE_DIR) ## Push the image for e2e tests + TARGET_PLATFORMS=$(ARCH) TAG=v0.0.1 CONTROLLER_IMAGE_NAME=turtles-e2e \ + ADDITIONAL_COMMANDS=$(CACHE_COMMANDS) $(MAKE) docker-build-and-push .PHONY: compile-e2e e2e-compile: ## Test e2e compilation