From fe43d5f27c7d7e6a746133ae6ede03de8015840b Mon Sep 17 00:00:00 2001 From: colmsnowplow Date: Wed, 10 Apr 2024 14:52:43 +0100 Subject: [PATCH] Add arm64 support --- .github/workflows/cd.yml | 6 ++- .github/workflows/ci.yml | 4 +- Dockerfile.aws | 6 ++- Dockerfile.main | 6 ++- Makefile | 85 ++++++++++++++++++++++------------ integration/docker-compose.yml | 7 +-- 6 files changed, 75 insertions(+), 39 deletions(-) diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index c0780a39..aa606a7d 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692 - go-version: ['1.21.1'] + go-version: ['1.22'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} env: @@ -109,10 +109,14 @@ jobs: tag_name: ${{ github.ref_name }} files: | build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_amd64.zip + build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_arm64.zip build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_amd64.zip + build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_arm64.zip build/compiled/snowbridge_${{ github.ref_name }}-aws-only_windows_amd64.zip build/compiled/snowbridge_${{ github.ref_name }}_darwin_amd64.zip + build/compiled/snowbridge_${{ github.ref_name }}_darwin_arm64.zip build/compiled/snowbridge_${{ github.ref_name }}_linux_amd64.zip + build/compiled/snowbridge_${{ github.ref_name }}_linux_arm64.zip build/compiled/snowbridge_${{ github.ref_name }}_windows_amd64.zip - name: Publish to DockerHub diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4c35c043..1c396e91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692 - go-version: ['1.21.1'] + go-version: ['1.22'] os: [ubuntu-latest] runs-on: ${{ matrix.os }} env: @@ -70,7 +70,7 @@ jobs: env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} - - name: Run Snyk to check for vulnerabilities in aws-only Docker + - name: Run Snyk to check for vulnerabilities in aws-only Docker image run: snyk container test snowplow/snowbridge:${{steps.version.outputs.VERSION}}-aws-only --severity-threshold=high env: SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} diff --git a/Dockerfile.aws b/Dockerfile.aws index bba70fe8..3b5cd8f5 100644 --- a/Dockerfile.aws +++ b/Dockerfile.aws @@ -1,7 +1,9 @@ -FROM alpine:3.18.5 +FROM --platform=$TARGETPLATFORM alpine:3.18.5 LABEL org.opencontainers.image.authors="Snowplow Analytics " -ADD build/output/linux/aws/cli/snowbridge /opt/snowplow/ +ARG TARGETARCH + +ADD build/output/linux/aws/cli/$TARGETARCH/snowbridge /opt/snowplow/ RUN adduser -D snowplow USER snowplow diff --git a/Dockerfile.main b/Dockerfile.main index 55c07feb..409c7f7f 100644 --- a/Dockerfile.main +++ b/Dockerfile.main @@ -1,7 +1,9 @@ -FROM alpine:3.18.5 +FROM --platform=$TARGETPLATFORM alpine:3.18.5 LABEL org.opencontainers.image.authors="Snowplow Analytics " -ADD build/output/linux/main/cli/snowbridge /opt/snowplow/ +ARG TARGETARCH + +ADD build/output/linux/main/cli/$TARGETARCH/snowbridge /opt/snowplow/ RUN adduser -D snowplow USER snowplow diff --git a/Makefile b/Makefile index 62576c98..4bf5049c 100644 --- a/Makefile +++ b/Makefile @@ -28,8 +28,9 @@ linux_out_dir = $(output_dir)/linux darwin_out_dir = $(output_dir)/darwin windows_out_dir = $(output_dir)/windows -container_name = snowplow/snowbridge +linux_container_image_out_dir = $(output_dir)/container/linux +container_name = snowplow/snowbridge # ----------------------------------------------------------------------------- # BUILDING @@ -48,44 +49,67 @@ cli: gox cli-linux cli-darwin cli-windows cp assets/awslicense/AMAZON_LICENSE AMAZON_LICENSE # linux aws: # Zip up the binaries - (cd $(linux_out_dir)/aws/cli/ && zip -r staging.zip snowbridge) + (cd $(linux_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge) # Add the readme, and relevant licence(s) - zip -u $(linux_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE + zip -u $(linux_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE # Move to its compiled_dir location, with appropriate long form name - mv $(linux_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip + mv $(linux_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip # Rinse and repeat for each distribution +# linux arm aws: + (cd $(linux_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge) + zip -u $(linux_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE + mv $(linux_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_arm64.zip # darwin aws: - (cd $(darwin_out_dir)/aws/cli/ && zip -r staging.zip snowbridge) - zip -u $(darwin_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE - mv $(darwin_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip + (cd $(darwin_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge) + zip -u $(darwin_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE + mv $(darwin_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip +# darwin arm aws: + (cd $(darwin_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge) + zip -u $(darwin_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE + mv $(darwin_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_arm64.zip # Windows aws: - (cd $(windows_out_dir)/aws/cli/ && zip -r staging.zip snowbridge.exe) - zip -u $(windows_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE - mv $(windows_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip + (cd $(windows_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge.exe) + zip -u $(windows_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE + mv $(windows_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip # linux main: - (cd $(linux_out_dir)/main/cli/ && zip -r staging.zip snowbridge) - zip -u $(linux_out_dir)/main/cli/staging.zip README.md LICENSE.md - mv $(linux_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip + (cd $(linux_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge) + zip -u $(linux_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md + mv $(linux_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip +# linux arm main: + (cd $(linux_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge) + zip -u $(linux_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md + mv $(linux_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_arm64.zip # darwin main: - (cd $(darwin_out_dir)/main/cli/ && zip -r staging.zip snowbridge) - zip -u $(darwin_out_dir)/main/cli/staging.zip README.md LICENSE.md - mv $(darwin_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip + (cd $(darwin_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge) + zip -u $(darwin_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md + mv $(darwin_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip +# darwin arm main: + (cd $(darwin_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge) + zip -u $(darwin_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md + mv $(darwin_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_arm64.zip # windows main: - (cd $(windows_out_dir)/main/cli/ && zip -r staging.zip snowbridge.exe) - zip -u $(windows_out_dir)/main/cli/staging.zip README.md LICENSE.md - mv $(windows_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip + (cd $(windows_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge.exe) + zip -u $(windows_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md + mv $(windows_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip +# Build CLI binaries for each distro cli-linux: gox - CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/ - CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/snowbridge ./cmd/main/cli/ + CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/ + CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/ + + CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/ + CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/ cli-darwin: gox - CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/ - CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/snowbridge ./cmd/main/cli/ + CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/ + CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/ + + CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/ + CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/ cli-windows: gox - CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/ - CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/snowbridge ./cmd/main/cli/ + CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/ + CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/ container: cli-linux docker build -t $(container_name):$(aws_only_version) -f Dockerfile.aws . @@ -122,7 +146,7 @@ test: test-setup integration-test: test-setup go test $(integration_test_dirs) -v -covermode=count -coverprofile=$(coverage_out) go tool cover -html=$(coverage_out) -o $(coverage_html) - go tool cover -func=$(coverage_out) + go tool cover -func=$(coverage_out) # e2e-test covers only the e2e release tests, in preparation for when these will rely on deployed assets e2e-test: test-setup @@ -141,7 +165,8 @@ e2e-down: integration-reset: integration-down integration-up -# For integration tests we need localstack and pubsub, but not kafka (yet) +# For integration tests we need localstack, pubsub kafka and http server +# To run on mac M1, for example, set the default docker platform: export DOCKER_DEFAULT_PLATFORM=linux/arm64 integration-up: http-up (cd $(integration_dir) && docker compose up -d) sleep 5 @@ -168,8 +193,10 @@ http-down: # Make & push docker assets, don't tag as latest if there's a `-` in the version (eg. 0.1.0-rc1) container-release: @-docker login --username $(DOCKER_USERNAME) --password $(DOCKER_PASSWORD) - docker push $(container_name):$(aws_only_version) - docker push $(container_name):$(version) + docker buildx create --name multi-arch-builder --driver=docker-container --platform linux/amd64,linux/arm64 --use + docker buildx build -t $(container_name):$(aws_only_version) -f Dockerfile.aws --platform=linux/amd64,linux/arm64 --push . + docker buildx build -t $(container_name):$(version) -f Dockerfile.main --platform=linux/amd64,linux/arm64 --push . + if ! [[ $(version) =~ "-" ]]; then docker tag ${container_name}:${version} ${container_name}:latest; docker push $(container_name):latest; fi; # ----------------------------------------------------------------------------- diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml index a1c1d075..5d834451 100644 --- a/integration/docker-compose.yml +++ b/integration/docker-compose.yml @@ -18,19 +18,20 @@ services: pubsub: image: bigtruedata/gcloud-pubsub-emulator + platform: "linux/amd64" # Only one platform exists for this image command: start --host-port 0.0.0.0:8432 ports: - "0.0.0.0:8432:8432" zookeeper: - image: confluentinc/cp-zookeeper:7.0.1 + image: confluentinc/cp-zookeeper:7.0.12 container_name: zookeeper environment: ZOOKEEPER_CLIENT_PORT: 2181 ZOOKEEPER_TICK_TIME: 2000 broker: - image: confluentinc/cp-kafka:7.0.1 + image: confluentinc/cp-kafka:7.0.12 container_name: broker ports: # To learn about configuring Kafka for access across networks see @@ -45,4 +46,4 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 \ No newline at end of file + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1