From 68f3b2c10510cbba54915476affeae3560c7da5a Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 15:32:00 -0800 Subject: [PATCH 01/12] Use native ARM builders for container builds This commit replaces #1088 and attempts to build a lading multi-platform image by use of organization provided ARM native builders. Unsure if this'll work but I figure it's worth a shot. Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 60 +++++++++++++++++++++++++++++---- Dockerfile | 4 +-- 2 files changed, 56 insertions(+), 8 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 2b6cd6f21..63526e0aa 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -8,8 +8,15 @@ env: IMAGE_NAME: ${{ github.repository }} jobs: - container: - runs-on: ubuntu-20.04 + build: + runs-on: ${{ matrix.os }} + strategy: + matrix: + include: + - arch: amd64 + runner: ubuntu-20.04 + - arch: arm64 + runner: arm-4core-linux-ubuntu24.04 permissions: contents: read packages: write @@ -47,9 +54,50 @@ jobs: with: file: Dockerfile builder: ${{ steps.buildx.outputs.name }} + platforms: linux/${{ matrix.arch }} + tags: | + ${{ steps.meta.outputs.tags }} + ${{ steps.meta.outputs.tags }}-${{ matrix.arch }} push: true - platforms: linux/amd64, linux/arm64 - tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha, scope=${{ github.workflow }} - cache-to: type=gha, scope=${{ github.workflow }} + cache-from: type=registry,ref=ghcr.io/datadog/lading:latest + cache-to: type=registry,ref=ghcr.io/datadog/lading:latest,mode=max + + manifest: + name: Create Multi-Arch Manifest + needs: build + runs-on: ubuntu-20.04 + permissions: + contents: read + packages: write + + steps: + - name: Log in to Container Registry + uses: docker/login-action@v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.7.1 + + - name: Extract Docker Metadata + uses: docker/metadata-action@v5.5.1 + id: meta + with: + tags: | + type=sha,format=long + type=ref,prefix=pr-,event=pr + type=semver,pattern={{version}},event=tag + type=semver,pattern={{major}}.{{minor}},event=tag + type=semver,pattern={{major}},event=tag + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Create and Push Multi-Arch Manifest + run: | + docker manifest create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }} \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }}-amd64 \ + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }}-arm64 + + docker manifest push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }} diff --git a/Dockerfile b/Dockerfile index f3b14bc1e..3bd5c0085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Update the rust version in-sync with the version in rust-toolchain.toml -FROM docker.io/rust:1.81.0-bullseye AS builder +FROM --platform=$BUILDPLATFORM docker.io/rust:1.81.0-bullseye AS builder RUN apt-get update && apt-get install -y \ protobuf-compiler fuse3 libfuse3-dev \ @@ -9,7 +9,7 @@ WORKDIR /app COPY . /app RUN cargo build --release --locked --bin lading -FROM docker.io/debian:bullseye-20240701-slim +FROM --platform=$BUILDPLATFORM docker.io/debian:bullseye-20240701-slim RUN apt-get update && apt-get install -y libfuse3-dev=3.10.3-2 fuse3=3.10.3-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/lading /usr/bin/lading From 90ae933400d47dee6706396a1d84f7d22b8483be Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 15:36:40 -0800 Subject: [PATCH 02/12] version tag ding Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 63526e0aa..9686dcc1f 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -73,17 +73,17 @@ jobs: steps: - name: Log in to Container Registry - uses: docker/login-action@v3.3.0 + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.7.1 + uses: docker/setup-buildx-action@vc47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Extract Docker Metadata - uses: docker/metadata-action@v5.5.1 + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 id: meta with: tags: | From f281619ff21793a96cc9eb16fcf21bcd287feb13 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 15:40:04 -0800 Subject: [PATCH 03/12] runs-on runner Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 9686dcc1f..5560f0aba 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -9,7 +9,7 @@ env: jobs: build: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.runner }} strategy: matrix: include: From 4d32b4d397e4510f547a23c04d184e7b7b9b60fa Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 15:42:33 -0800 Subject: [PATCH 04/12] actionlint ding Signed-off-by: Brian L. Troutwine --- .github/actionlint.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/actionlint.yml diff --git a/.github/actionlint.yml b/.github/actionlint.yml new file mode 100644 index 000000000..aa633e7fe --- /dev/null +++ b/.github/actionlint.yml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - arm-4core-linux-ubuntu24.04 From c5987a13695b2407ea9c978cf0e5d1c80eb67d1f Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 15:51:30 -0800 Subject: [PATCH 05/12] left a 'v' in the tag Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 5560f0aba..425c7a522 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -80,7 +80,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@vc47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - name: Extract Docker Metadata uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 From 06f248f192e14d7889a9120d64e456149edf51db Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 16:19:22 -0800 Subject: [PATCH 06/12] rely only on tags, use cache tag for caching Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 425c7a522..7d8dc75d9 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -60,8 +60,8 @@ jobs: ${{ steps.meta.outputs.tags }}-${{ matrix.arch }} push: true labels: ${{ steps.meta.outputs.labels }} - cache-from: type=registry,ref=ghcr.io/datadog/lading:latest - cache-to: type=registry,ref=ghcr.io/datadog/lading:latest,mode=max + cache-from: type=registry,ref=ghcr.io/datadog/lading:cache + cache-to: type=registry,ref=ghcr.io/datadog/lading:cache,mode=max manifest: name: Create Multi-Arch Manifest @@ -96,8 +96,8 @@ jobs: - name: Create and Push Multi-Arch Manifest run: | - docker manifest create ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }} \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }}-amd64 \ - ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }}-arm64 + docker manifest create ${{ steps.meta.outputs.tags }} \ + ${{ steps.meta.outputs.tags }}-amd64 \ + ${{ steps.meta.outputs.tags }}-arm64 - docker manifest push ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.tags }} + docker manifest push ${{ steps.meta.outputs.tags }} From b0d510335fcb515683249ec4bfa5ab62244c2875 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 16:39:21 -0800 Subject: [PATCH 07/12] container tag or something Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 7d8dc75d9..89eb86406 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -88,10 +88,6 @@ jobs: with: tags: | type=sha,format=long - type=ref,prefix=pr-,event=pr - type=semver,pattern={{version}},event=tag - type=semver,pattern={{major}}.{{minor}},event=tag - type=semver,pattern={{major}},event=tag images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Create and Push Multi-Arch Manifest From ef91a481840a6bd193f4fcf709a841c4b4df97ba Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 16:57:57 -0800 Subject: [PATCH 08/12] single platform images Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 89eb86406..e3bbc23b4 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -53,11 +53,7 @@ jobs: uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 with: file: Dockerfile - builder: ${{ steps.buildx.outputs.name }} - platforms: linux/${{ matrix.arch }} - tags: | - ${{ steps.meta.outputs.tags }} - ${{ steps.meta.outputs.tags }}-${{ matrix.arch }} + tags: ${{ steps.meta.outputs.tags }}-${{ matrix.arch }} push: true labels: ${{ steps.meta.outputs.labels }} cache-from: type=registry,ref=ghcr.io/datadog/lading:cache From 5054e5cac4b18b4277a58e16d3a210d7bc0baf79 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 17:12:47 -0800 Subject: [PATCH 09/12] remove more platform Signed-off-by: Brian L. Troutwine --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3bd5c0085..f3b14bc1e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Update the rust version in-sync with the version in rust-toolchain.toml -FROM --platform=$BUILDPLATFORM docker.io/rust:1.81.0-bullseye AS builder +FROM docker.io/rust:1.81.0-bullseye AS builder RUN apt-get update && apt-get install -y \ protobuf-compiler fuse3 libfuse3-dev \ @@ -9,7 +9,7 @@ WORKDIR /app COPY . /app RUN cargo build --release --locked --bin lading -FROM --platform=$BUILDPLATFORM docker.io/debian:bullseye-20240701-slim +FROM docker.io/debian:bullseye-20240701-slim RUN apt-get update && apt-get install -y libfuse3-dev=3.10.3-2 fuse3=3.10.3-2 && rm -rf /var/lib/apt/lists/* COPY --from=builder /app/target/release/lading /usr/bin/lading From 5f4fce9634ae90c985a78f1f07a39e4104932532 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 17:33:02 -0800 Subject: [PATCH 10/12] annotations??? Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 66 ++++++++++++++++++--------------- 1 file changed, 37 insertions(+), 29 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index e3bbc23b4..025e814e1 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -59,37 +59,45 @@ jobs: cache-from: type=registry,ref=ghcr.io/datadog/lading:cache cache-to: type=registry,ref=ghcr.io/datadog/lading:cache,mode=max - manifest: - name: Create Multi-Arch Manifest - needs: build - runs-on: ubuntu-20.04 - permissions: - contents: read - packages: write +manifest: + name: Create Multi-Arch Manifest + needs: build + runs-on: ubuntu-20.04 + permissions: + contents: read + packages: write - steps: - - name: Log in to Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Log in to Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - name: Extract Docker Metadata - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 - id: meta - with: - tags: | - type=sha,format=long - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + - name: Extract Docker Metadata + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + id: meta + with: + tags: | + type=sha,format=long + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Confirm images exist + run: | + docker pull ${{ steps.meta.outputs.tags }}-amd64 + docker pull ${{ steps.meta.outputs.tags }}-arm64 + + - name: Create and Push Multiarch Manifest + run: | + docker manifest create ${{ steps.meta.outputs.tags }} \ + ${{ steps.meta.outputs.tags }}-amd64 \ + ${{ steps.meta.outputs.tags }}-arm64 - - name: Create and Push Multi-Arch Manifest - run: | - docker manifest create ${{ steps.meta.outputs.tags }} \ - ${{ steps.meta.outputs.tags }}-amd64 \ - ${{ steps.meta.outputs.tags }}-arm64 + docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-amd64 --os linux --arch amd64 + docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-arm64 --os linux --arch arm64 - docker manifest push ${{ steps.meta.outputs.tags }} + docker manifest push ${{ steps.meta.outputs.tags }} From 9491c1a9fb101f51c1806e09551ac1ab0a1075f9 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 17:33:57 -0800 Subject: [PATCH 11/12] indent Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 84 ++++++++++++++++----------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 025e814e1..24e254609 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -59,45 +59,45 @@ jobs: cache-from: type=registry,ref=ghcr.io/datadog/lading:cache cache-to: type=registry,ref=ghcr.io/datadog/lading:cache,mode=max -manifest: - name: Create Multi-Arch Manifest - needs: build - runs-on: ubuntu-20.04 - permissions: - contents: read - packages: write - - steps: - - name: Log in to Container Registry - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 - with: - registry: ${{ env.REGISTRY }} - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Extract Docker Metadata - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 - id: meta - with: - tags: | - type=sha,format=long - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - - name: Confirm images exist - run: | - docker pull ${{ steps.meta.outputs.tags }}-amd64 - docker pull ${{ steps.meta.outputs.tags }}-arm64 - - - name: Create and Push Multiarch Manifest - run: | - docker manifest create ${{ steps.meta.outputs.tags }} \ - ${{ steps.meta.outputs.tags }}-amd64 \ - ${{ steps.meta.outputs.tags }}-arm64 - - docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-amd64 --os linux --arch amd64 - docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-arm64 --os linux --arch arm64 - - docker manifest push ${{ steps.meta.outputs.tags }} + manifest: + name: Create Multi-Arch Manifest + needs: build + runs-on: ubuntu-20.04 + permissions: + contents: read + packages: write + + steps: + - name: Log in to Container Registry + uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 + + - name: Extract Docker Metadata + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1 + id: meta + with: + tags: | + type=sha,format=long + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Confirm images exist + run: | + docker pull ${{ steps.meta.outputs.tags }}-amd64 + docker pull ${{ steps.meta.outputs.tags }}-arm64 + + - name: Create and Push Multiarch Manifest + run: | + docker manifest create ${{ steps.meta.outputs.tags }} \ + ${{ steps.meta.outputs.tags }}-amd64 \ + ${{ steps.meta.outputs.tags }}-arm64 + + docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-amd64 --os linux --arch amd64 + docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-arm64 --os linux --arch arm64 + + docker manifest push ${{ steps.meta.outputs.tags }} From 290323440c8ce7eee22be90a8d9d46e1e29d2c55 Mon Sep 17 00:00:00 2001 From: "Brian L. Troutwine" Date: Wed, 6 Nov 2024 18:32:23 -0800 Subject: [PATCH 12/12] use docker buildx imagetools Signed-off-by: Brian L. Troutwine --- .github/workflows/container.yml | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/.github/workflows/container.yml b/.github/workflows/container.yml index 24e254609..e327f4edd 100644 --- a/.github/workflows/container.yml +++ b/.github/workflows/container.yml @@ -86,18 +86,9 @@ jobs: type=sha,format=long images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - - name: Confirm images exist - run: | - docker pull ${{ steps.meta.outputs.tags }}-amd64 - docker pull ${{ steps.meta.outputs.tags }}-arm64 - - name: Create and Push Multiarch Manifest run: | - docker manifest create ${{ steps.meta.outputs.tags }} \ + docker buildx imagetools create \ + --tag ${{ steps.meta.outputs.tags }} \ ${{ steps.meta.outputs.tags }}-amd64 \ ${{ steps.meta.outputs.tags }}-arm64 - - docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-amd64 --os linux --arch amd64 - docker manifest annotate ${{ steps.meta.outputs.tags }} ${{ steps.meta.outputs.tags }}-arm64 --os linux --arch arm64 - - docker manifest push ${{ steps.meta.outputs.tags }}