From 06a36ba9b453bcb02cc7af22c4210969864c0938 Mon Sep 17 00:00:00 2001 From: Mohamed Mahmoud Date: Fri, 31 May 2024 08:42:32 -0400 Subject: [PATCH] Add mutli arch build and manifest for the operator repo Signed-off-by: Mohamed Mahmoud --- .github/workflows/image-build.yaml | 21 +++++++- .github/workflows/pull_request.yml | 23 ++++++--- Containerfile.bpfman-agent | 9 ++-- Containerfile.bpfman-operator | 9 ++-- Makefile | 77 +++++++++++++++++++++++------- config/test/kustomization.yaml | 2 +- 6 files changed, 110 insertions(+), 31 deletions(-) diff --git a/.github/workflows/image-build.yaml b/.github/workflows/image-build.yaml index 3dd7f7c6b..0f85ff45a 100644 --- a/.github/workflows/image-build.yaml +++ b/.github/workflows/image-build.yaml @@ -17,6 +17,10 @@ jobs: id-token: write # needed for signing the images with GitHub OIDC Token runs-on: ubuntu-latest + env: + WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent + WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator + WF_MULTIARCH_TARGETS: amd64 arm64 ppc64le s390x strategy: fail-fast: false matrix: @@ -71,7 +75,7 @@ jobs: if: ${{ matrix.image.build_language == 'go' }} with: # prettier-ignore - go-version: '1.21' # yamllint disable-line rule:quoted-strings + go-version: '1.22' # yamllint disable-line rule:quoted-strings - uses: sigstore/cosign-installer@v3.5.0 @@ -120,3 +124,18 @@ jobs: for tag in ${tags[@]}; do cosign sign -y "${tag}@${{ steps.push-image.outputs.digest }}" done + - name: get short sha + run: | + echo "short_sha=$(git rev-parse --short HEAD)" >> $GITHUB_ENV + - name: Build and Push multi arch images + run: | + IMAGE_TAG=${{ env.short_sha }}" + BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }} + BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}} + MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make build-images push-images + - name: Build and Push manifest + run: | + IMAGE_TAG=${{ env.short_sha }}" + BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }} + BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}} + MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make manifest-build manifest-push diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index d440f7590..3b66c02e4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ['1.21','1.22'] + go: ['1.22'] steps: - name: install make run: sudo apt-get install make @@ -27,7 +27,7 @@ jobs: - name: check clean vendors run: go mod vendor - name: Report coverage - if: ${{ matrix.go == '1.21' }} + if: ${{ matrix.go == '1.22' }} uses: codecov/codecov-action@v4 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} @@ -41,10 +41,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: ['1.21'] + go: ['1.22'] env: - BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent:int-test - BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator:int-test + WF_BPFMAN_AGENT_IMG: quay.io/bpfman/bpfman-agent + WF_BPFMAN_OPERATOR_IMG: quay.io/bpfman/bpfman-operator + WF_MULTIARCH_TARGETS: amd64 XDP_PASS_PRIVATE_IMAGE_CREDS: ${{ secrets.XDP_PASS_PRIVATE_IMAGE_CREDS }} steps: - name: Check disk space @@ -79,10 +80,18 @@ jobs: fetch-depth: 0 - name: build images - run: make build-images + run: | + IMAGE_TAG="int-test" + BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}" + BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}" + MULTIARCH_TARGETS="${{ env.WF_MULTIARCH_TARGETS }}" make build-images - name: run integration tests - run: make test-integration + run: | + IMAGE_TAG="int-test" + BPFMAN_AGENT_IMG="${{ env.WF_BPFMAN_AGENT_IMG }}-${{ env.WF_MULTIARCH_TARGETS }}" + BPFMAN_OPERATOR_IMG="${{ env.WF_BPFMAN_OPERATOR_IMG}}-${{ env.WF_MULTIARCH_TARGETS }}" + make test-integration - name: Check disk space run: df -h diff --git a/Containerfile.bpfman-agent b/Containerfile.bpfman-agent index e04265d18..17bb20cb6 100644 --- a/Containerfile.bpfman-agent +++ b/Containerfile.bpfman-agent @@ -1,7 +1,10 @@ # Build the manager binary -FROM golang:1.22 as bpfman-agent-build +ARG TARGETPLATFORM=linux/amd64 +ARG BUILDPLATFORM=linux/amd64 +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-agent-build ARG TARGETOS -ARG TARGETARCH +ARG TARGETPLATFORM +ARG TARGETARCH=amd64 WORKDIR /usr/src/bpfman-operator @@ -28,7 +31,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven # Use the fedora minimal image to reduce the size of the final image but still # be able to easily install extra packages. -FROM quay.io/fedora/fedora-minimal +FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal ARG DNF_CMD="microdnf" # The full fedora image can be used for debugging purposes. To use it, comment diff --git a/Containerfile.bpfman-operator b/Containerfile.bpfman-operator index 97f4f0460..45b8597ad 100644 --- a/Containerfile.bpfman-operator +++ b/Containerfile.bpfman-operator @@ -1,7 +1,10 @@ # Build the manager binary -FROM golang:1.22 as bpfman-operator-build +ARG TARGETPLATFORM=linux/amd64 +ARG BUILDPLATFORM=linux/amd64 +FROM --platform=$BUILDPLATFORM docker.io/library/golang:1.22 as bpfman-operator-build ARG TARGETOS -ARG TARGETARCH +ARG TARGETPLATFORM +ARG TARGETARCH=amd64 WORKDIR /usr/src/bpfman-operator @@ -32,7 +35,7 @@ RUN CGO_ENABLED=0 GOOS=${TARGETOS:-linux} GOARCH=${TARGETARCH} go build -mod ven # The full fedora image can be used for debugging purposes, but you may need to # change "microdnf" to "dnf" below to install extra packages. # FROM fedora:38 -FROM quay.io/fedora/fedora-minimal +FROM --platform=$TARGETPLATFORM quay.io/fedora/fedora-minimal ARG TARGETARCH WORKDIR / COPY --from=bpfman-operator-build /usr/src/bpfman-operator/config/bpfman-deployment/daemonset.yaml ./config/bpfman-deployment/daemonset.yaml diff --git a/Makefile b/Makefile index 75dd9a484..65bf6dc5e 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2) # - use environment variables to overwrite this value (e.g export VERSION=0.0.2) VERSION ?= 0.4.1 +MULTIARCH_TARGETS ?= amd64 # CHANNELS define the bundle channels used in the bundle. # Add a new line here if you would like to change its default config. (E.g CHANNELS = "candidate,fast,stable") @@ -45,12 +46,12 @@ USE_IMAGE_DIGESTS ?= false ifeq ($(USE_IMAGE_DIGESTS), true) BUNDLE_GEN_FLAGS += --use-image-digests endif - +IMAGE_TAG ?= latest # Image URL to use all building/pushing image targets -BPFMAN_IMG ?= quay.io/bpfman/bpfman:latest -BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:latest -BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:latest -BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:latest +BPFMAN_IMG ?= quay.io/bpfman/bpfman:$(IMAGE_TAG) +BPFMAN_AGENT_IMG ?= quay.io/bpfman/bpfman-agent:$(IMAGE_TAG) +BPFMAN_OPERATOR_IMG ?= quay.io/bpfman/bpfman-operator:$(IMAGE_TAG) +BPFMAN_OPERATOR_BUNDLE_IMG ?= quay.io/bpfman/bpfman-operator-bundle:$(IMAGE_TAG) KIND_CLUSTER_NAME ?= bpfman-deployment # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary. @@ -71,6 +72,23 @@ endif SHELL = /usr/bin/env bash -o pipefail .SHELLFLAGS = -ec +# Image building tool (docker / podman) - docker is preferred in CI +OCI_BIN_PATH := $(shell which docker 2>/dev/null || which podman) +OCI_BIN ?= $(shell basename ${OCI_BIN_PATH}) + +# build a single arch target provided as argument +define build_target + echo 'building $(1) for arch $(2)'; \ + DOCKER_BUILDKIT=1 $(OCI_BIN) buildx build --load --build-arg TARGETPLATFORM=linux/$(2)\ + --build-arg TARGETARCH=$(2) --build-arg BUILDPLATFORM=linux/amd64 -t $(1)-$(2) -f $(3) .; +endef + +# push a single arch target image +define push_target + echo 'pushing image $(1)-$(2)'; \ + DOCKER_BUILDKIT=1 $(OCI_BIN) push $(1)-$(2); +endef + .PHONY: all all: build @@ -126,7 +144,8 @@ OPERATOR_SDK_DL_URL=https://github.com/operator-framework/operator-sdk/releases/ .PHONY: operator-sdk operator-sdk: $(OPERATOR_SDK) $(OPERATOR_SDK): $(LOCALBIN) - test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} && mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; } + test -s $(LOCALBIN)/operator_sdk || { curl -LO ${OPERATOR_SDK_DL_URL} && chmod +x ${OPERATOR_SDK_DL_NAME} &&\ + mv ${OPERATOR_SDK_DL_NAME} $(LOCALBIN)/operator-sdk; } .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. @@ -219,7 +238,8 @@ generate-typed-clients: client-gen ## Generate typed client code .PHONY: generate-typed-listers generate-typed-listers: lister-gen ## Generate typed listers code - $(LISTER_GEN) "${PKG}/apis/v1alpha1" \ + $(LISTER_GEN) \ + "${PKG}/apis/v1alpha1" \ --output-pkg "${PKG}/pkg/client" \ --output-dir "./pkg/client" \ ${COMMON_FLAGS} @@ -227,7 +247,8 @@ generate-typed-listers: lister-gen ## Generate typed listers code .PHONY: generate-typed-informers generate-typed-informers: informer-gen ## Generate typed informers code - $(INFORMER_GEN) "${PKG}/apis/v1alpha1" \ + $(INFORMER_GEN) \ + "${PKG}/apis/v1alpha1" \ --versioned-clientset-package "${PKG}/pkg/client/clientset" \ --listers-package "${PKG}/pkg/client" \ --output-pkg "${PKG}/pkg/client" \ @@ -279,7 +300,8 @@ test-integration: ## Run Integration tests. .PHONY: bundle bundle: operator-sdk generate kustomize manifests ## Generate bundle manifests and metadata, then validate generated files. cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG} - cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} + cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} &&\ + $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} $(KUSTOMIZE) build config/manifests | $(OPERATOR_SDK) generate bundle $(BUNDLE_GEN_FLAGS) cp config/manifests/dependencies.yaml bundle/metadata/ $(OPERATOR_SDK) bundle validate ./bundle @@ -299,14 +321,35 @@ build: fmt ## Build bpfman-operator and bpfman-agent binaries. # (i.e. docker build --platform linux/arm64 ). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: build-images -build-images: ## Build bpfman, bpfman-agent, and bpfman-operator images. - docker build -t ${BPFMAN_OPERATOR_IMG} -f Containerfile.bpfman-operator ./ - docker build -t ${BPFMAN_AGENT_IMG} -f Containerfile.bpfman-agent ./ +build-images: ## Build bpfman-agent, and bpfman-operator images. + trap 'exit' INT; \ + $(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_OPERATOR_IMG},$(target),Containerfile.bpfman-operator)) \ + $(foreach target,$(MULTIARCH_TARGETS),$(call build_target,${BPFMAN_AGENT_IMG},$(target),Containerfile.bpfman-agent)) .PHONY: push-images push-images: ## Push bpfman-agent, bpfman-operator images. - docker push ${BPFMAN_OPERATOR_IMG} - docker push ${BPFMAN_AGENT_IMG} + trap 'exit' INT; \ + $(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_OPERATOR_IMG},$(target))) \ + $(foreach target,$(MULTIARCH_TARGETS),$(call push_target,${BPFMAN_AGENT_IMG},$(target))) + +.PHONY: manifest-build +manifest-build: ## Build MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent. + echo 'building manifest for $(BPFMAN_OPERATOR_IMG) and $(BPFMAN_AGENT_IMG)' + DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_OPERATOR_IMG} -f + DOCKER_BUILDKIT=1 $(OCI_BIN) rmi ${BPFMAN_AGENT_IMG} -f + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_OPERATOR_IMG} $(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_OPERATOR_IMG}-$(target)); + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest create ${BPFMAN_AGENT_IMG} $(foreach target,$(MULTIARCH_TARGETS), --amend ${BPFMAN_AGENT_IMG}-$(target)); + +.PHONY: manifest-push +manifest-push: ## Push MULTIARCH_TARGETS manifest for bpfman-operator and bpfman-agent. + @echo 'publish manifest for $(BPFMAN_OPERATOR_IMG) and $(BPFMAN_AGENT_IMG)' +ifeq (${OCI_BIN}, docker) + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG}; + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG}; +else + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_OPERATOR_IMG} docker://${BPFMAN_OPERATOR_IMG}; + DOCKER_BUILDKIT=1 $(OCI_BIN) manifest push ${BPFMAN_AGENT_IMG} docker://${BPFMAN_AGENT_IMG}; +endif .PHONY: load-images-kind load-images-kind: ## Load bpfman-agent, and bpfman-operator images into the running local kind devel cluster. @@ -370,7 +413,8 @@ destroy-kind: ## Destroy Kind cluster .PHONY: deploy deploy: manifests kustomize ## Deploy bpfman-operator to the K8s cluster specified in ~/.kube/config with the csi driver initialized. cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG} - cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} + cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && \ + $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} $(KUSTOMIZE) build config/default | kubectl apply -f - .PHONY: undeploy @@ -392,7 +436,8 @@ run-on-kind: kustomize setup-kind build-images load-images-kind deploy ## Kind D .PHONY: deploy-openshift deploy-openshift: manifests kustomize ## Deploy bpfman-operator to the Openshift cluster specified in ~/.kube/config. cd config/bpfman-operator-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-operator=${BPFMAN_OPERATOR_IMG} - cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} + cd config/bpfman-deployment && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman=${BPFMAN_IMG} \ + && $(KUSTOMIZE) edit set image quay.io/bpfman/bpfman-agent=${BPFMAN_AGENT_IMG} $(KUSTOMIZE) build config/openshift | kubectl apply -f - .PHONY: undeploy-openshift diff --git a/config/test/kustomization.yaml b/config/test/kustomization.yaml index 5f52dab86..a34862e5b 100644 --- a/config/test/kustomization.yaml +++ b/config/test/kustomization.yaml @@ -46,4 +46,4 @@ resources: images: - name: quay.io/bpfman/bpfman-operator newName: quay.io/bpfman/bpfman-operator - newTag: int-test + newTag: int-test-amd64