Merge pull request #173 from zhaofengli/multi-arch-image

Build and push multi-arch images

.ci/build-and-push-images.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [[ "$#" -lt "2" ]]; then
+	>&2 echo "Usage: $0 <image name> <tag1> ..."
+	>&2 echo "Example: $0 ghcr.io/zhaofengli/attic main abcd123"
+	exit 1
+fi
+
+cleanup() {
+	if [[ -f "${manifest_spec}" ]]; then
+		rm "${manifest_spec}"
+	fi
+}
+trap cleanup EXIT
+
+image_name="$1"
+tags=("${@:2}")
+
+manifest_spec="$(mktemp -t attic-manifest-spec.XXXXXXXXXX)"
+
+declare -a digests
+
+emit_header() {
+	echo "image: ${image_name}"
+	echo "tags:"
+	for tag in "${tags[@]}"; do
+		echo "- ${tag}"
+	done
+	echo "manifests:"
+}
+
+push_digest() {
+	source_image="docker-archive:$1"
+	digest="$(skopeo inspect "${source_image}" | jq -r .Digest)"
+	target_image="docker://${image_name}@${digest}"
+
+	>&2 echo "${source_image} ▸ ${target_image}"
+	>&2 skopeo copy --insecure-policy "${source_image}" "${target_image}"
+
+	echo -n "- "
+	skopeo inspect "${source_image}" | \
+		jq '{platform: {architecture: .Architecture, os: .Os}, image: ($image_name + "@" + .Digest)}' \
+		--arg image_name "${image_name}"
+}
+
+>>"${manifest_spec}" emit_header
+
+nix build .#attic-server-image .#attic-server-image-aarch64 -L --print-out-paths | \
+while read -r output; do
+	>>"${manifest_spec}" push_digest "${output}"
+done
+
+>&2 echo "----------"
+>&2 echo "Generated manifest-tool spec:"
+>&2 echo "----------"
+cat "${manifest_spec}"
+>&2 echo "----------"
+
+manifest-tool push from-spec "${manifest_spec}"

.github/workflows/book.yml

@@ -16,9 +16,9 @@ jobs:
     if: github.repository == 'zhaofengli/attic'
 
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic
@@ -40,12 +40,12 @@ jobs:
           cp --recursive --dereference --no-preserve=mode,ownership result public
 
       - name: Upload book artifact
-        uses: actions/upload-pages-artifact@v2.0.0
+        uses: actions/upload-pages-artifact@v3.0.1
         with:
           path: public
 
       - name: Deploy book
-        uses: actions/deploy-pages@v3.0.1
+        uses: actions/deploy-pages@v4.0.5
 
       # TODO: Just take a diff of the list of store paths, also abstract all of this out
       - name: Push build artifacts

.github/workflows/build.yml

@@ -4,7 +4,7 @@ on:
   push:
 env:
   REGISTRY: ghcr.io
-  IMAGE_NAME: ${{ github.repository }}
+  IMAGE_NAME: ghcr.io/${{ github.repository }}
 jobs:
   tests:
     strategy:
@@ -17,18 +17,15 @@ jobs:
           - "2.24"
           - "default"
     runs-on: ${{ matrix.os }}
-    permissions:
-      contents: read
-      packages: write
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
       - name: Install current Bash on macOS
         if: runner.os == 'macOS'
         run: |
           command -v brew && brew install bash || true
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic
@@ -38,6 +35,7 @@ jobs:
           fi
 
       - name: Configure Attic
+        continue-on-error: true
         run: |
           : "${ATTIC_SERVER:=https://staging.attic.rs/}"
           : "${ATTIC_CACHE:=attic-ci}"
@@ -75,30 +73,88 @@ jobs:
               .#internalMatrix."$system".\"${{ matrix.nix }}\".cargoArtifacts \
             | xargs attic push "ci:$ATTIC_CACHE"
           fi
+
+  image:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push'
+    needs:
+      - tests
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@v4.1.7
+
+      - name: Install current Bash on macOS
+        if: runner.os == 'macOS'
+        run: |
+          command -v brew && brew install bash || true
+
+      - uses: DeterminateSystems/nix-installer-action@v14
+        continue-on-error: true # Self-hosted runners already have Nix installed
+
+      - name: Install Attic
+        run: |
+          if ! command -v attic &> /dev/null; then
+            ./.github/install-attic-ci.sh
+          fi
+
+      - name: Configure Attic
+        continue-on-error: true
+        run: |
+          : "${ATTIC_SERVER:=https://staging.attic.rs/}"
+          : "${ATTIC_CACHE:=attic-ci}"
+          echo ATTIC_CACHE=$ATTIC_CACHE >>$GITHUB_ENV
+          export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
+          attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN"
+          attic use "$ATTIC_CACHE"
+        env:
+          ATTIC_SERVER: ${{ secrets.ATTIC_SERVER }}
+          ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }}
+          ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }}
+
+      - name: Cache dev shell
+        run: |
+          .ci/cache-shell.sh
+          system=$(nix-instantiate --eval -E 'builtins.currentSystem')
+          echo system=$system >>$GITHUB_ENV
+
       - name: Log in to the Container registry
-        uses: docker/login-action@v3.0.0
-        if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+        uses: docker/login-action@v3.3.0
         with:
           registry: ${{ env.REGISTRY }}
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Push build container image
-        if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch)
+      - name: Build and push container images
         continue-on-error: true
         run: |
-          IMAGE_ID=ghcr.io/${IMAGE_NAME}
-          TARBALL=$(nix build --json .#attic-server-image | jq -r '.[].outputs.out')
-          BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
-          TAG="${{ github.sha }}"
-          [[ "${{ github.ref }}" == "refs/tags/"* ]] && TAG=$(echo $BRANCH | sed -e 's/^v//')
-          docker load < ${TARBALL}
-          echo IMAGE_ID=$IMAGE_ID
-          echo TAG=$TAG
-          docker tag attic-server:main "${IMAGE_ID}:${TAG}"
-          docker push ${IMAGE_ID}:${TAG}
-          if [ "$BRANCH" == "main" ]; then
-            TAG="latest"
-            docker tag attic-server:main "${IMAGE_ID}:${TAG}"
-            docker push ${IMAGE_ID}:${TAG}
+          declare -a tags
+          tags+=("${{ github.sha }}")
+
+          branch=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,')
+          if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then
+            tags+=("$(echo $branch | sed -e 's/^v//')")
+          else
+            tags+=("${branch}")
+          fi
+
+          if [ "$branch" == "${{ github.event.repository.default_branch }}" ]; then
+            tags+=("latest")
+          fi
+
+          >&2 echo "Image: ${IMAGE_NAME}"
+          >&2 echo "Tags: ${tags[@]}"
+
+          .ci/run just ci-build-and-push-images "${IMAGE_NAME}" "${tags[@]}"
+
+      # TODO: Just take a diff of the list of store paths, also abstract all of this out
+      - name: Push build artifacts
+        run: |
+          export PATH=$HOME/.nix-profile/bin:$PATH # FIXME
+          if [ -n "$ATTIC_TOKEN" ]; then
+            nix build --no-link --print-out-paths -L \
+              .#attic-server-image \
+              .#attic-server-image-aarch64 \
+            | xargs attic push "ci:$ATTIC_CACHE"
           fi

.github/workflows/lint.yml

@@ -10,9 +10,14 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v4.1.1
+      - uses: actions/checkout@v4.1.7
 
-      - uses: DeterminateSystems/nix-installer-action@v9
+      - name: Install current Bash on macOS
+        if: runner.os == 'macOS'
+        run: |
+          command -v brew && brew install bash || true
+
+      - uses: DeterminateSystems/nix-installer-action@v14
         continue-on-error: true # Self-hosted runners already have Nix installed
 
       - name: Install Attic

crane.nix

@@ -7,8 +7,9 @@
 
 { stdenv
 , lib
+, buildPackages
 , craneLib
-, rustPlatform
+, rust
 , runCommand
 , writeReferencesToFile
 , pkg-config
@@ -19,12 +20,22 @@
 , boost
 , darwin
 , libiconv
+
+, extraPackageArgs ? {}
 }:
 
 let
   version = "0.1.0";
 
-  ignoredPaths = [ ".github" "target" "book" "nixos" "integration-tests" ];
+  ignoredPaths = [
+    ".ci"
+    ".github"
+    "book"
+    "flake"
+    "integration-tests"
+    "nixos"
+    "target"
+  ];
 
   src = lib.cleanSourceWith {
     filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) ignoredPaths);
@@ -43,7 +54,19 @@ let
     libiconv
   ];
 
-  cargoArtifacts = craneLib.buildDepsOnly {
+  crossArgs = let
+    rustTargetSpec = rust.toRustTargetSpec stdenv.hostPlatform;
+    rustTargetSpecEnv = lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] rustTargetSpec);
+  in lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) {
+    depsBuildBuild = [ buildPackages.stdenv.cc ];
+
+    CARGO_BUILD_TARGET = rustTargetSpec;
+    "CARGO_TARGET_${rustTargetSpecEnv}_LINKER" = "${stdenv.cc.targetPrefix}cc";
+  };
+
+  extraArgs = crossArgs // extraPackageArgs;
+
+  cargoArtifacts = craneLib.buildDepsOnly ({
     pname = "attic";
     inherit src version nativeBuildInputs buildInputs;
 
@@ -54,7 +77,7 @@ let
     # With `use-zstd`, the cargo artifacts are archived in a `tar.zstd`. This is
     # actually set if you use `buildPackage` without passing `cargoArtifacts`.
     installCargoArtifactsMode = "use-zstd";
-  };
+  } // extraArgs);
 
   mkAttic = args: craneLib.buildPackage ({
     pname = "attic";
@@ -86,7 +109,7 @@ let
       maintainers = with maintainers; [ zhaofengli ];
       platforms = platforms.linux ++ platforms.darwin;
     };
-  } // args);
+  } // args // extraArgs);
 
   attic = mkAttic {
     cargoExtraArgs = "-p attic-client -p attic-server";
@@ -106,7 +129,7 @@ let
   #
   # We don't enable fat LTO in the default `attic` package since it
   # dramatically increases build time.
-  attic-server = craneLib.buildPackage {
+  attic-server = craneLib.buildPackage ({
     pname = "attic-server";
 
     # We don't pull in the common cargoArtifacts because the feature flags
@@ -120,13 +143,13 @@ let
 
     CARGO_PROFILE_RELEASE_LTO = "fat";
     CARGO_PROFILE_RELEASE_CODEGEN_UNITS = "1";
-  };
+  } // extraArgs);
 
   # Attic interacts with Nix directly and its tests require trusted-user access
   # to nix-daemon to import NARs, which is not possible in the build sandbox.
   # In the CI pipeline, we build the test executable inside the sandbox, then
   # run it outside.
-  attic-tests = craneLib.mkCargoDerivation {
+  attic-tests = craneLib.mkCargoDerivation ({
     pname = "attic-tests";
 
     inherit src version buildInputs cargoArtifacts;
@@ -151,7 +174,7 @@ let
 
       runHook postInstall
     '';
-  };
+  } // extraArgs);
 in {
   inherit cargoArtifacts attic attic-client attic-server attic-tests;
 }

flake/devshells.nix

@@ -35,7 +35,7 @@ in
       cfg = config.attic.devshell;
     in {
       attic.devshell.packageSets = with pkgs; {
-        rustc = [
+        rustc = lib.optionals (config.attic.toolchain == null) [
           rustc
         ];
 
@@ -63,6 +63,8 @@ in
           sqlite-interactive
 
           flyctl
+          skopeo
+          manifest-tool
         ] ++ lib.optionals pkgs.stdenv.isLinux [
           wrangler
         ];