Add arm64 support

snowplow · Apr 10, 2024 · fe43d5f · fe43d5f
1 parent 3eefedc
commit fe43d5f
Show file tree

Hide file tree

Showing 6 changed files with 75 additions and 39 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -11,7 +11,7 @@ jobs:
     strategy:
       matrix:
         # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692
-        go-version: ['1.21.1']
+        go-version: ['1.22']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
@@ -109,10 +109,14 @@ jobs:
         tag_name: ${{ github.ref_name }}
         files: |
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}-aws-only_darwin_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}-aws-only_linux_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}-aws-only_windows_amd64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_darwin_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}_darwin_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_linux_amd64.zip
+          build/compiled/snowbridge_${{ github.ref_name }}_linux_arm64.zip
           build/compiled/snowbridge_${{ github.ref_name }}_windows_amd64.zip
 
     - name: Publish to DockerHub

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -13,7 +13,7 @@ jobs:
     strategy:
       matrix:
         # Quotes are required: https://github.com/actions/setup-go/issues/326#issuecomment-1415719692
-        go-version: ['1.21.1']
+        go-version: ['1.22']
         os: [ubuntu-latest]
     runs-on: ${{ matrix.os }}
     env:
@@ -70,7 +70,7 @@ jobs:
       env:
         SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
-    - name: Run Snyk to check for vulnerabilities in aws-only Docker
+    - name: Run Snyk to check for vulnerabilities in aws-only Docker image
       run: snyk container test snowplow/snowbridge:${{steps.version.outputs.VERSION}}-aws-only --severity-threshold=high
       env:
         SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

diff --git a/Dockerfile.aws b/Dockerfile.aws
@@ -1,7 +1,9 @@
-FROM alpine:3.18.5
+FROM --platform=$TARGETPLATFORM alpine:3.18.5
 LABEL org.opencontainers.image.authors="Snowplow Analytics <support@snowplow.io>"
 
-ADD build/output/linux/aws/cli/snowbridge /opt/snowplow/
+ARG TARGETARCH
+
+ADD build/output/linux/aws/cli/$TARGETARCH/snowbridge /opt/snowplow/
 
 RUN adduser -D snowplow
 USER snowplow

diff --git a/Dockerfile.main b/Dockerfile.main
@@ -1,7 +1,9 @@
-FROM alpine:3.18.5
+FROM --platform=$TARGETPLATFORM alpine:3.18.5
 LABEL org.opencontainers.image.authors="Snowplow Analytics <support@snowplow.io>"
 
-ADD build/output/linux/main/cli/snowbridge /opt/snowplow/
+ARG TARGETARCH
+
+ADD build/output/linux/main/cli/$TARGETARCH/snowbridge /opt/snowplow/
 
 RUN adduser -D snowplow
 USER snowplow

diff --git a/Makefile b/Makefile
@@ -28,8 +28,9 @@ linux_out_dir   = $(output_dir)/linux
 darwin_out_dir  = $(output_dir)/darwin
 windows_out_dir = $(output_dir)/windows
 
-container_name = snowplow/snowbridge
+linux_container_image_out_dir = $(output_dir)/container/linux
 
+container_name = snowplow/snowbridge
 
 # -----------------------------------------------------------------------------
 #  BUILDING
@@ -48,44 +49,67 @@ cli: gox cli-linux cli-darwin cli-windows
 	cp assets/awslicense/AMAZON_LICENSE AMAZON_LICENSE
 # linux aws:
 # Zip up the binaries
-	(cd $(linux_out_dir)/aws/cli/ && zip -r staging.zip snowbridge)
+	(cd $(linux_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge)
 # Add the readme, and relevant licence(s)
-	zip -u $(linux_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	zip -u $(linux_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
 # Move to its compiled_dir location, with appropriate long form name
-	mv $(linux_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip
+	mv $(linux_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_amd64.zip
 # Rinse and repeat for each distribution
+# linux arm aws:
+	(cd $(linux_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(linux_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_linux_arm64.zip
 # darwin aws:
-	(cd $(darwin_out_dir)/aws/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(darwin_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
-	mv $(darwin_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip
+	(cd $(darwin_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(darwin_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_amd64.zip
+# darwin arm aws:
+	(cd $(darwin_out_dir)/aws/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/aws/cli/arm64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(darwin_out_dir)/aws/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_darwin_arm64.zip
 # Windows aws:
-	(cd $(windows_out_dir)/aws/cli/ && zip -r staging.zip snowbridge.exe)
-	zip -u $(windows_out_dir)/aws/cli/staging.zip README.md LICENSE.md AMAZON_LICENSE
-	mv $(windows_out_dir)/aws/cli/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip
+	(cd $(windows_out_dir)/aws/cli/amd64/ && zip -r staging.zip snowbridge.exe)
+	zip -u $(windows_out_dir)/aws/cli/amd64/staging.zip README.md LICENSE.md AMAZON_LICENSE
+	mv $(windows_out_dir)/aws/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(aws_only_version)_windows_amd64.zip
 # linux main:
-	(cd $(linux_out_dir)/main/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(linux_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(linux_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip
+	(cd $(linux_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(linux_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_amd64.zip
+# linux arm main:
+	(cd $(linux_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(linux_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md
+	mv $(linux_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_linux_arm64.zip
 # darwin main:
-	(cd $(darwin_out_dir)/main/cli/ && zip -r staging.zip snowbridge)
-	zip -u $(darwin_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(darwin_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip
+	(cd $(darwin_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(darwin_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_amd64.zip
+# darwin arm main:
+	(cd $(darwin_out_dir)/main/cli/arm64/ && zip -r staging.zip snowbridge)
+	zip -u $(darwin_out_dir)/main/cli/arm64/staging.zip README.md LICENSE.md
+	mv $(darwin_out_dir)/main/cli/arm64/staging.zip $(compiled_dir)/snowbridge_$(version)_darwin_arm64.zip	
 # windows main:
-	(cd $(windows_out_dir)/main/cli/ && zip -r staging.zip snowbridge.exe)
-	zip -u $(windows_out_dir)/main/cli/staging.zip README.md LICENSE.md
-	mv $(windows_out_dir)/main/cli/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip
+	(cd $(windows_out_dir)/main/cli/amd64/ && zip -r staging.zip snowbridge.exe)
+	zip -u $(windows_out_dir)/main/cli/amd64/staging.zip README.md LICENSE.md
+	mv $(windows_out_dir)/main/cli/amd64/staging.zip $(compiled_dir)/snowbridge_$(version)_windows_amd64.zip
 
+# Build CLI binaries for each distro
 cli-linux: gox
-	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=linux/amd64 -output=$(linux_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
+
+	CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=linux/arm64 -output=$(linux_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/	
 
 cli-darwin: gox
-	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/amd64 -output=$(darwin_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
+
+	CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/aws/cli/arm64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=darwin/arm64 -output=$(darwin_out_dir)/main/cli/arm64/snowbridge ./cmd/main/cli/	
 
 cli-windows: gox
-	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/snowbridge ./cmd/aws/cli/
-	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/snowbridge ./cmd/main/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/aws/cli/amd64/snowbridge ./cmd/aws/cli/
+	CGO_ENABLED=0 gox -osarch=windows/amd64 -output=$(windows_out_dir)/main/cli/amd64/snowbridge ./cmd/main/cli/
 
 container: cli-linux
 	docker build -t $(container_name):$(aws_only_version) -f Dockerfile.aws .
@@ -122,7 +146,7 @@ test: test-setup
 integration-test: test-setup
 	go test $(integration_test_dirs) -v -covermode=count -coverprofile=$(coverage_out)
 	go tool cover -html=$(coverage_out) -o $(coverage_html)
-	go tool cover -func=$(coverage_out)
+	go tool cover -func=$(coverage_out)	
 
 # e2e-test covers only the e2e release tests, in preparation for when these will rely on deployed assets
 e2e-test: test-setup
@@ -141,7 +165,8 @@ e2e-down:
 
 integration-reset: integration-down integration-up
 
-# For integration tests we need localstack and pubsub, but not kafka (yet)
+# For integration tests we need localstack, pubsub kafka and http server
+# To run on mac M1, for example, set the default docker platform: export DOCKER_DEFAULT_PLATFORM=linux/arm64
 integration-up: http-up
 	(cd $(integration_dir) && docker compose up -d)
 	sleep 5
@@ -168,8 +193,10 @@ http-down:
 # Make & push docker assets, don't tag as latest if there's a `-` in the version (eg. 0.1.0-rc1)
 container-release:
 	@-docker login --username $(DOCKER_USERNAME) --password $(DOCKER_PASSWORD)
-	docker push $(container_name):$(aws_only_version)
-	docker push $(container_name):$(version)
+	docker buildx create --name multi-arch-builder --driver=docker-container --platform linux/amd64,linux/arm64 --use
+	docker buildx build -t $(container_name):$(aws_only_version) -f Dockerfile.aws --platform=linux/amd64,linux/arm64 --push .
+	docker buildx build -t $(container_name):$(version) -f Dockerfile.main --platform=linux/amd64,linux/arm64 --push .
+
 	if ! [[ $(version) =~ "-" ]]; then docker tag ${container_name}:${version} ${container_name}:latest; docker push $(container_name):latest; fi;
 
 # -----------------------------------------------------------------------------

diff --git a/integration/docker-compose.yml b/integration/docker-compose.yml
@@ -18,19 +18,20 @@ services:
 
   pubsub:
     image: bigtruedata/gcloud-pubsub-emulator
+    platform: "linux/amd64" # Only one platform exists for this image
     command: start --host-port 0.0.0.0:8432
     ports:
       - "0.0.0.0:8432:8432"
 
   zookeeper:
-    image: confluentinc/cp-zookeeper:7.0.1
+    image: confluentinc/cp-zookeeper:7.0.12
     container_name: zookeeper
     environment:
       ZOOKEEPER_CLIENT_PORT: 2181
       ZOOKEEPER_TICK_TIME: 2000
 
   broker:
-    image: confluentinc/cp-kafka:7.0.1
+    image: confluentinc/cp-kafka:7.0.12
     container_name: broker
     ports:
     # To learn about configuring Kafka for access across networks see
@@ -45,4 +46,4 @@ services:
       KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092,PLAINTEXT_INTERNAL://broker:29092
       KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
       KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
-      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1