diff --git a/.ci/build-and-push-images.sh b/.ci/build-and-push-images.sh new file mode 100755 index 0000000..740d9f9 --- /dev/null +++ b/.ci/build-and-push-images.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$#" -lt "2" ]]; then + >&2 echo "Usage: $0 ..." + >&2 echo "Example: $0 ghcr.io/zhaofengli/attic main abcd123" + exit 1 +fi + +cleanup() { + if [[ -f "${manifest_spec}" ]]; then + rm "${manifest_spec}" + fi +} +trap cleanup EXIT + +image_name="$1" +tags=("${@:2}") + +manifest_spec="$(mktemp -t attic-manifest-spec.XXXXXXXXXX)" + +declare -a digests + +emit_header() { + echo "image: ${image_name}" + echo "tags:" + for tag in "${tags[@]}"; do + echo "- ${tag}" + done + echo "manifests:" +} + +push_digest() { + source_image="docker-archive:$1" + digest="$(skopeo inspect "${source_image}" | jq -r .Digest)" + target_image="docker://${image_name}@${digest}" + + >&2 echo "${source_image} ▸ ${target_image}" + >&2 skopeo copy --insecure-policy "${source_image}" "${target_image}" + + echo -n "- " + skopeo inspect "${source_image}" | \ + jq '{platform: {architecture: .Architecture, os: .Os}, image: ($image_name + "@" + .Digest)}' \ + --arg image_name "${image_name}" +} + +>>"${manifest_spec}" emit_header + +nix build .#attic-server-image .#attic-server-image-aarch64 -L --print-out-paths | \ +while read -r output; do + >>"${manifest_spec}" push_digest "${output}" +done + +>&2 echo "----------" +>&2 echo "Generated manifest-tool spec:" +>&2 echo "----------" +cat "${manifest_spec}" +>&2 echo "----------" + +manifest-tool push from-spec "${manifest_spec}" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 4fdd709..2a31963 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -16,9 +16,9 @@ jobs: if: github.repository == 'zhaofengli/attic' steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v4.1.7 - - uses: DeterminateSystems/nix-installer-action@v9 + - uses: DeterminateSystems/nix-installer-action@v14 continue-on-error: true # Self-hosted runners already have Nix installed - name: Install Attic @@ -40,12 +40,12 @@ jobs: cp --recursive --dereference --no-preserve=mode,ownership result public - name: Upload book artifact - uses: actions/upload-pages-artifact@v2.0.0 + uses: actions/upload-pages-artifact@v3.0.1 with: path: public - name: Deploy book - uses: actions/deploy-pages@v3.0.1 + uses: actions/deploy-pages@v4.0.5 # TODO: Just take a diff of the list of store paths, also abstract all of this out - name: Push build artifacts diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 031d200..f9a62a7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,7 +4,7 @@ on: push: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ghcr.io/${{ github.repository }} jobs: tests: strategy: @@ -17,18 +17,15 @@ jobs: - "2.24" - "default" runs-on: ${{ matrix.os }} - permissions: - contents: read - packages: write steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v4.1.7 - name: Install current Bash on macOS if: runner.os == 'macOS' run: | command -v brew && brew install bash || true - - uses: DeterminateSystems/nix-installer-action@v9 + - uses: DeterminateSystems/nix-installer-action@v14 continue-on-error: true # Self-hosted runners already have Nix installed - name: Install Attic @@ -38,6 +35,7 @@ jobs: fi - name: Configure Attic + continue-on-error: true run: | : "${ATTIC_SERVER:=https://staging.attic.rs/}" : "${ATTIC_CACHE:=attic-ci}" @@ -75,30 +73,88 @@ jobs: .#internalMatrix."$system".\"${{ matrix.nix }}\".cargoArtifacts \ | xargs attic push "ci:$ATTIC_CACHE" fi + + image: + runs-on: ubuntu-latest + if: github.event_name == 'push' + needs: + - tests + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@v4.1.7 + + - name: Install current Bash on macOS + if: runner.os == 'macOS' + run: | + command -v brew && brew install bash || true + + - uses: DeterminateSystems/nix-installer-action@v14 + continue-on-error: true # Self-hosted runners already have Nix installed + + - name: Install Attic + run: | + if ! command -v attic &> /dev/null; then + ./.github/install-attic-ci.sh + fi + + - name: Configure Attic + continue-on-error: true + run: | + : "${ATTIC_SERVER:=https://staging.attic.rs/}" + : "${ATTIC_CACHE:=attic-ci}" + echo ATTIC_CACHE=$ATTIC_CACHE >>$GITHUB_ENV + export PATH=$HOME/.nix-profile/bin:$PATH # FIXME + attic login --set-default ci "$ATTIC_SERVER" "$ATTIC_TOKEN" + attic use "$ATTIC_CACHE" + env: + ATTIC_SERVER: ${{ secrets.ATTIC_SERVER }} + ATTIC_CACHE: ${{ secrets.ATTIC_CACHE }} + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + + - name: Cache dev shell + run: | + .ci/cache-shell.sh + system=$(nix-instantiate --eval -E 'builtins.currentSystem') + echo system=$system >>$GITHUB_ENV + - name: Log in to the Container registry - uses: docker/login-action@v3.0.0 - if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + uses: docker/login-action@v3.3.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - - name: Push build container image - if: runner.os == 'Linux' && github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) + - name: Build and push container images continue-on-error: true run: | - IMAGE_ID=ghcr.io/${IMAGE_NAME} - TARBALL=$(nix build --json .#attic-server-image | jq -r '.[].outputs.out') - BRANCH=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') - TAG="${{ github.sha }}" - [[ "${{ github.ref }}" == "refs/tags/"* ]] && TAG=$(echo $BRANCH | sed -e 's/^v//') - docker load < ${TARBALL} - echo IMAGE_ID=$IMAGE_ID - echo TAG=$TAG - docker tag attic-server:main "${IMAGE_ID}:${TAG}" - docker push ${IMAGE_ID}:${TAG} - if [ "$BRANCH" == "main" ]; then - TAG="latest" - docker tag attic-server:main "${IMAGE_ID}:${TAG}" - docker push ${IMAGE_ID}:${TAG} + declare -a tags + tags+=("${{ github.sha }}") + + branch=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + if [[ "${{ github.ref }}" == "refs/tags/"* ]]; then + tags+=("$(echo $branch | sed -e 's/^v//')") + else + tags+=("${branch}") + fi + + if [ "$branch" == "${{ github.event.repository.default_branch }}" ]; then + tags+=("latest") + fi + + >&2 echo "Image: ${IMAGE_NAME}" + >&2 echo "Tags: ${tags[@]}" + + .ci/run just ci-build-and-push-images "${IMAGE_NAME}" "${tags[@]}" + + # TODO: Just take a diff of the list of store paths, also abstract all of this out + - name: Push build artifacts + run: | + export PATH=$HOME/.nix-profile/bin:$PATH # FIXME + if [ -n "$ATTIC_TOKEN" ]; then + nix build --no-link --print-out-paths -L \ + .#attic-server-image \ + .#attic-server-image-aarch64 \ + | xargs attic push "ci:$ATTIC_CACHE" fi diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index a52a61f..4ae17ac 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,9 +10,14 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4.1.1 + - uses: actions/checkout@v4.1.7 - - uses: DeterminateSystems/nix-installer-action@v9 + - name: Install current Bash on macOS + if: runner.os == 'macOS' + run: | + command -v brew && brew install bash || true + + - uses: DeterminateSystems/nix-installer-action@v14 continue-on-error: true # Self-hosted runners already have Nix installed - name: Install Attic diff --git a/crane.nix b/crane.nix index 4ec2827..ffe9eb0 100644 --- a/crane.nix +++ b/crane.nix @@ -7,8 +7,9 @@ { stdenv , lib +, buildPackages , craneLib -, rustPlatform +, rust , runCommand , writeReferencesToFile , pkg-config @@ -19,12 +20,22 @@ , boost , darwin , libiconv + +, extraPackageArgs ? {} }: let version = "0.1.0"; - ignoredPaths = [ ".github" "target" "book" "nixos" "integration-tests" ]; + ignoredPaths = [ + ".ci" + ".github" + "book" + "flake" + "integration-tests" + "nixos" + "target" + ]; src = lib.cleanSourceWith { filter = name: type: !(type == "directory" && builtins.elem (baseNameOf name) ignoredPaths); @@ -43,7 +54,19 @@ let libiconv ]; - cargoArtifacts = craneLib.buildDepsOnly { + crossArgs = let + rustTargetSpec = rust.toRustTargetSpec stdenv.hostPlatform; + rustTargetSpecEnv = lib.toUpper (builtins.replaceStrings [ "-" ] [ "_" ] rustTargetSpec); + in lib.optionalAttrs (stdenv.hostPlatform != stdenv.buildPlatform) { + depsBuildBuild = [ buildPackages.stdenv.cc ]; + + CARGO_BUILD_TARGET = rustTargetSpec; + "CARGO_TARGET_${rustTargetSpecEnv}_LINKER" = "${stdenv.cc.targetPrefix}cc"; + }; + + extraArgs = crossArgs // extraPackageArgs; + + cargoArtifacts = craneLib.buildDepsOnly ({ pname = "attic"; inherit src version nativeBuildInputs buildInputs; @@ -54,7 +77,7 @@ let # With `use-zstd`, the cargo artifacts are archived in a `tar.zstd`. This is # actually set if you use `buildPackage` without passing `cargoArtifacts`. installCargoArtifactsMode = "use-zstd"; - }; + } // extraArgs); mkAttic = args: craneLib.buildPackage ({ pname = "attic"; @@ -86,7 +109,7 @@ let maintainers = with maintainers; [ zhaofengli ]; platforms = platforms.linux ++ platforms.darwin; }; - } // args); + } // args // extraArgs); attic = mkAttic { cargoExtraArgs = "-p attic-client -p attic-server"; @@ -106,7 +129,7 @@ let # # We don't enable fat LTO in the default `attic` package since it # dramatically increases build time. - attic-server = craneLib.buildPackage { + attic-server = craneLib.buildPackage ({ pname = "attic-server"; # We don't pull in the common cargoArtifacts because the feature flags @@ -120,13 +143,13 @@ let CARGO_PROFILE_RELEASE_LTO = "fat"; CARGO_PROFILE_RELEASE_CODEGEN_UNITS = "1"; - }; + } // extraArgs); # Attic interacts with Nix directly and its tests require trusted-user access # to nix-daemon to import NARs, which is not possible in the build sandbox. # In the CI pipeline, we build the test executable inside the sandbox, then # run it outside. - attic-tests = craneLib.mkCargoDerivation { + attic-tests = craneLib.mkCargoDerivation ({ pname = "attic-tests"; inherit src version buildInputs cargoArtifacts; @@ -151,7 +174,7 @@ let runHook postInstall ''; - }; + } // extraArgs); in { inherit cargoArtifacts attic attic-client attic-server attic-tests; } diff --git a/flake/devshells.nix b/flake/devshells.nix index 2452021..e7b1145 100644 --- a/flake/devshells.nix +++ b/flake/devshells.nix @@ -35,7 +35,7 @@ in cfg = config.attic.devshell; in { attic.devshell.packageSets = with pkgs; { - rustc = [ + rustc = lib.optionals (config.attic.toolchain == null) [ rustc ]; @@ -63,6 +63,8 @@ in sqlite-interactive flyctl + skopeo + manifest-tool ] ++ lib.optionals pkgs.stdenv.isLinux [ wrangler ]; diff --git a/flake/packages.nix b/flake/packages.nix index 8cd10e7..c67bb4c 100644 --- a/flake/packages.nix +++ b/flake/packages.nix @@ -1,96 +1,161 @@ -{ self, inputs, lib, makeCranePkgs, ... }: +{ self +, lib +, flake-parts-lib +, inputs +, config +, makeCranePkgs +, getSystem +, ... +}: + let - defaultMakeCranePkgs = pkgs: let - craneLib = inputs.crane.mkLib pkgs; - in pkgs.callPackage ../crane.nix { inherit craneLib; }; + inherit (lib) + mkOption + types + ; + inherit (flake-parts-lib) + mkPerSystemOption + ; + + # Re-evaluate perSystem with cross nixpkgs + # HACK before https://github.com/hercules-ci/flake-parts/issues/95 is solved + evalCross = { system, pkgs }: config.allSystems.${system}.debug.extendModules { + modules = [ + ({ config, lib, ... }: { + _module.args.pkgs = pkgs; + _module.args.self' = lib.mkForce config; + }) + ]; + }; in { - _module.args.makeCranePkgs = lib.mkDefault defaultMakeCranePkgs; + options = { + perSystem = mkPerSystemOption { + options.attic = { + toolchain = mkOption { + type = types.nullOr types.package; + default = null; + }; + extraPackageArgs = mkOption { + type = types.attrsOf types.anything; + default = {}; + }; + }; + }; + }; - perSystem = { self', pkgs, cranePkgs, ... }: (lib.mkMerge [ - { - _module.args.cranePkgs = makeCranePkgs pkgs; + config = { + _module.args.makeCranePkgs = lib.mkDefault (pkgs: let + perSystemConfig = getSystem pkgs.system; + craneLib = builtins.foldl' (acc: f: f acc) pkgs [ + inputs.crane.mkLib + (craneLib: + if perSystemConfig.attic.toolchain == null then craneLib + else craneLib.overrideToolchain config.attic.toolchain + ) + ]; + in pkgs.callPackage ../crane.nix { + inherit craneLib; + inherit (perSystemConfig.attic) extraPackageArgs; + }); - packages = { - default = self'.packages.attic; + perSystem = { self', pkgs, config, cranePkgs, ... }: (lib.mkMerge [ + { + _module.args.cranePkgs = makeCranePkgs pkgs; - inherit (cranePkgs) - attic - attic-client - attic-server - ; + packages = { + default = self'.packages.attic; - attic-nixpkgs = pkgs.callPackage ../package.nix { }; + inherit (cranePkgs) + attic + attic-client + attic-server + ; - attic-ci-installer = pkgs.callPackage ../ci-installer.nix { - inherit self; - }; + attic-nixpkgs = pkgs.callPackage ../package.nix { }; - book = pkgs.callPackage ../book { - attic = self'.packages.attic; + attic-ci-installer = pkgs.callPackage ../ci-installer.nix { + inherit self; + }; + + book = pkgs.callPackage ../book { + attic = self'.packages.attic; + }; }; - }; - } - - (lib.mkIf pkgs.stdenv.isLinux { - packages = { - attic-server-image = pkgs.dockerTools.buildImage { - name = "attic-server"; - tag = "main"; - copyToRoot = [ - self'.packages.attic-server - - # Debugging utilities for `fly ssh console` - pkgs.busybox - - # Now required by the fly.io sshd - pkgs.dockerTools.fakeNss - ]; - config = { - Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ]; - Env = [ - "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + } + + (lib.mkIf pkgs.stdenv.isLinux { + packages = { + attic-server-image = pkgs.dockerTools.buildImage { + name = "attic-server"; + tag = "main"; + copyToRoot = [ + self'.packages.attic-server + + # Debugging utilities for `fly ssh console` + pkgs.busybox + + # Now required by the fly.io sshd + pkgs.dockerTools.fakeNss ]; + config = { + Entrypoint = [ "${self'.packages.attic-server}/bin/atticd" ]; + Env = [ + "SSL_CERT_FILE=${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" + ]; + }; }; }; - }; - }) - - # Unfortunately, x86_64-darwin fails to evaluate static builds - (lib.mkIf (pkgs.system != "x86_64-darwin") { - packages = { - # TODO: Make this work with Crane - attic-static = (pkgs.pkgsStatic.callPackage ../package.nix { - nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { - patches = (old.patches or []) ++ [ - # To be submitted - (pkgs.fetchpatch { - url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; - hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; - }) + }) + + (lib.mkIf (pkgs.system == "x86_64-linux") { + packages = { + attic-server-image-aarch64 = let + eval = evalCross { + system = "aarch64-linux"; + pkgs = pkgs.pkgsCross.aarch64-multiplatform; + }; + + in eval.config.packages.attic-server-image; + }; + }) + + # Unfortunately, x86_64-darwin fails to evaluate static builds + (lib.mkIf (pkgs.system != "x86_64-darwin") { + packages = { + # TODO: Make this work with Crane + attic-static = (pkgs.pkgsStatic.callPackage ../package.nix { + nix = pkgs.pkgsStatic.nix.overrideAttrs (old: { + patches = (old.patches or []) ++ [ + # To be submitted + (pkgs.fetchpatch { + url = "https://github.com/NixOS/nix/compare/3172c51baff5c81362fcdafa2e28773c2949c660...6b09a02536d5946458b537dfc36b7d268c9ce823.diff"; + hash = "sha256-LFLq++J2XitEWQ0o57ihuuUlYk2PgUr11h7mMMAEe3c="; + }) + ]; + }); + }).overrideAttrs (old: { + nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ + pkgs.nukeReferences ]; - }); - }).overrideAttrs (old: { - nativeBuildInputs = (old.nativeBuildInputs or []) ++ [ - pkgs.nukeReferences - ]; - # Read by pkg_config crate (do some autodetection in build.rs?) - PKG_CONFIG_ALL_STATIC = "1"; + # Read by pkg_config crate (do some autodetection in build.rs?) + PKG_CONFIG_ALL_STATIC = "1"; - "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; - RUSTFLAGS = "-C relocation-model=static"; + "NIX_CFLAGS_LINK_${pkgs.pkgsStatic.stdenv.cc.suffixSalt}" = "-lc"; + RUSTFLAGS = "-C relocation-model=static"; - postFixup = (old.postFixup or "") + '' - rm -f $out/nix-support/propagated-build-inputs - nuke-refs $out/bin/attic - ''; - }); + postFixup = (old.postFixup or "") + '' + rm -f $out/nix-support/propagated-build-inputs + nuke-refs $out/bin/attic + ''; + }); - attic-client-static = self'.packages.attic-static.override { - clientOnly = true; + attic-client-static = self'.packages.attic-static.override { + clientOnly = true; + }; }; - }; - }) - ]); + }) + ]); + }; } diff --git a/justfile b/justfile index 30f4368..ffe06e4 100644 --- a/justfile +++ b/justfile @@ -45,3 +45,7 @@ ci-unit-tests matrix: # (CI) Run rustfmt check ci-rustfmt: cargo fmt --check + +# (CI) Build and push images +ci-build-and-push-images *args: + .ci/build-and-push-images.sh {{ args }}