From 79fb32585241706ea283a1672cacf39e0845f3ae Mon Sep 17 00:00:00 2001 From: Zhaofeng Li Date: Mon, 9 Sep 2024 16:59:04 -0400 Subject: [PATCH] Build and push multi-arch images Fixes #147. --- .ci/build-and-push-images.sh | 61 ++++++++++++++++++++++ .github/workflows/build.yml | 98 ++++++++++++++++++++++++++++-------- justfile | 4 ++ 3 files changed, 142 insertions(+), 21 deletions(-) create mode 100755 .ci/build-and-push-images.sh diff --git a/.ci/build-and-push-images.sh b/.ci/build-and-push-images.sh new file mode 100755 index 0000000..643303e --- /dev/null +++ b/.ci/build-and-push-images.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -lt "2" ]]; then + >&2 echo "Usage: $0 ..." + >&2 echo "Example: $0 ghcr.io/zhaofengli/attic main abcd123" + exit 1 +fi + +cleanup() { + if [[ -f "${manifest_spec}" ]]; then + rm "${manifest_spec}" + fi +} +trap cleanup EXIT + +image_name="$1" +tags=("${@:2}") + +manifest_spec="$(mktemp -t attic-manifest-spec.XXXXXXXXXX)" + +declare -a digests + +emit_header() { + echo "image: ${image_name}" + echo "tags:" + for tag in "${tags[@]}"; do + echo "- ${tag}" + done + echo "manifests:" +} + +push_digest() { + source_image="docker-archive:$1" + digest="$(skopeo inspect "${source_image}" | jq -r .Digest)" + target_image="docker://${image_name}@${digest}" + + >&2 echo "${source_image} ▸ ${target_image}" + >&2 skopeo copy --insecure-policy "${source_image}" "${target_image}" + + echo -n "- " + skopeo inspect "${source_image}" | \ + jq '{platform: {architecture: .Architecture, os: .Os}, image: ($image_name + "@" + .Digest)}' \ + --arg image_name "${image_name}" +} + +>>"${manifest_spec}" emit_header + +nix build .#attic-server-image -L -o image +nix build .#attic-server-image-aarch64 -L -o arm64-image + +>>"${manifest_spec}" push_digest ./image +>>"${manifest_spec}" push_digest ./arm64-image + +>&2 echo "----------" +>&2 echo "Generated manifest-tool spec:" +>&2 echo "----------" +cat "${manifest_spec}" +>&2 echo "----------" + +manifest-tool push from-spec "${manifest_spec}" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 031d200..990907e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ghcr.io/${{ github.repository }} jobs: tests: strategy: @@ -17,9 +17,6 @@ jobs: - "2.24" - "default" runs-on: ${{ matrix.os }} - permissions: - contents: read - packages: write steps: - uses: actions/checkout@v4.1.1 @@ -38,6 +35,7 @@ jobs: fi - name: Configure Attic + continue-on-error: true run: | : "${ATTIC_SERVER:=https://staging.attic.rs/}" : "${ATTIC_CACHE:=attic-ci}" @@ -75,30 +73,88 @@ jobs: .#internalMatrix."$system".\"${{ matrix.nix }}\".cargoArtifacts \ | xargs attic push "ci:$ATTIC_CACHE" fi + + image: + runs-on: ubuntu-latest + if: github.event_name == 'push' + needs: + - tests + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4.1.1 + + - name: Install current Bash on macOS + if: runner.os == 'macOS' + run: | + command -v brew && brew install bash || true + + - uses: DeterminateSystems/nix-installer-action@v9 + continue-on-error: true # Self-hosted runners already have Nix installed + + - name: Install Attic + run: | + if ! command -v attic &> /dev/null; then + ./.github/install-attic-ci.sh + fi + + - name: Configure Attic + continue-on-error: true + run: | + : "${ATTIC_SERVER:=https://staging.attic.rs/}" + : "${ATTIC_CACHE:=attic-ci}" + echo ATTIC_CACHE=$ATTIC_CACHE >>$GITHUB_ENV + export PATH=$HOME/.nix-profile/bin:$PATH # FIXME + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" + attic use "$ATTIC_CACHE" + env: + ATTIC_SERVER: ${{ secrets.ATTIC_SERVER }} + ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }} + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + + - name: Cache dev shell + run: | + .ci/cache-shell.sh + system=$(nix-instantiate --eval -E 'builtins.currentSystem') + echo system=$system >>$GITHUB_ENV + - name: Log in to the Container registry uses: docker/login-action@v3.0.0 - if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Push build container image - if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + - name: Build and push container images continue-on-error: true run: | - IMAGE_ID=ghcr.io/${IMAGE_NAME} - TARBALL=$(nix build --json .#attic-server-image | jq -r '.[].outputs.out') - BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - TAG="${{ github.sha }}" - [[ "${{ github.ref }}" == "refs/tags/"* ]] && TAG=$(echo $BRANCH | sed -e 's/^v//') - docker load < ${TARBALL} - echo IMAGE_ID=$IMAGE_ID - echo TAG=$TAG - docker tag attic-server:main "${IMAGE_ID}:${TAG}" - docker push ${IMAGE_ID}:${TAG} - if [ "$BRANCH" == "main" ]; then - TAG="latest" - docker tag attic-server:main "${IMAGE_ID}:${TAG}" - docker push ${IMAGE_ID}:${TAG} + declare -a tags + tags+=("${{ github.sha }}") + + branch=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then + tags+=("$(echo $branch | sed -e 's/^v//')") + else + tags+=("${branch}") + fi + + if [ "$branch" == "${{ github.event.repository.default_branch }}" ]; then + tags+=("latest") + fi + + >&2 echo "Image: ${IMAGE_NAME}" + >&2 echo "Tags: ${tags[@]}" + + .ci/run just ci-build-and-push-images "${IMAGE_NAME}" "${tags[@]}" + + # TODO: Just take a diff of the list of store paths, also abstract all of this out + - name: Push build artifacts + run: | + export PATH=$HOME/.nix-profile/bin:$PATH # FIXME + if [ -n "$ATTIC_TOKEN" ]; then + nix build --no-link --print-out-paths -L \ + .#attic-server-image \ + .#attic-server-image-aarch64 \ + | xargs attic push "ci:$ATTIC_CACHE" fi diff --git a/justfile b/justfile index 30f4368..ffe06e4 100644 --- a/justfile +++ b/justfile @@ -45,3 +45,7 @@ ci-unit-tests matrix: # (CI) Run rustfmt check ci-rustfmt: cargo fmt --check + +# (CI) Build and push images +ci-build-and-push-images *args: + .ci/build-and-push-images.sh {{ args }}