From 65273d9ebd851a55c0502a3954ad66fdeb71754e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Somhairle=20MacLe=C3=B2id?= Date: Wed, 15 Feb 2023 18:10:14 +0000 Subject: [PATCH] Add a script to repeatably generate `linux-arm64` and `darwin-arm64` builds (#380) * Simplify Builds * Copy out caching * Copy out caching * Safer rm * Use pnpm * Add ARM check --- .gitignore | 4 +++ Dockerfile.release | 17 +++++++++++ RELEASE.md | 2 +- build-npm-package.sh | 71 -------------------------------------------- build-releases.sh | 22 ++++++++++++++ package.json | 1 + pnpm-lock.yaml | 7 +++++ 7 files changed, 52 insertions(+), 72 deletions(-) create mode 100644 Dockerfile.release delete mode 100644 build-npm-package.sh create mode 100755 build-releases.sh diff --git a/.gitignore b/.gitignore index 9513c91738f..935895ae7c0 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,7 @@ node_modules /rust-project.json # Directory where clangd puts its indexing work /.cache/ + +/.bazel-cache + +/workerd-* diff --git a/Dockerfile.release b/Dockerfile.release new file mode 100644 index 00000000000..d99fbe5ef6f --- /dev/null +++ b/Dockerfile.release @@ -0,0 +1,17 @@ +FROM node:bullseye AS builder + +WORKDIR /workerd + +RUN apt-get update +RUN apt-get install -y curl clang libc++-dev libc++abi-dev + +COPY . . +COPY .bazel-cache /bazel-disk-cache +RUN npm install -g pnpm +RUN pnpm install + +RUN pnpm exec bazelisk build --disk_cache=/bazel-disk-cache -c opt //src/workerd/server:workerd + +FROM scratch as artifact +COPY --from=builder /workerd/bazel-bin/src/workerd/server/workerd /workerd-linux-arm64 +COPY --from=builder /bazel-disk-cache /.bazel-cache diff --git a/RELEASE.md b/RELEASE.md index c53681a8ac7..d8f07806169 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -9,7 +9,7 @@ The primary distribution channel for `workerd` right now is through `npm`. We us ## Releases -This is pretty simple, and completely automatic—every time the compatibility date in [compatibility-date.capnp](src/workerd/io/compatibility-date.capnp) changes, a new release is generated, along with the built binaries for `linux-64` and `darwin-64`. This is governed by the [release.yml](.github/workflows/release.yml) GitHub Action. Since this only generates binaries for `darwin-64` and `linux-64`, binaries for `darwin-arm64` and `linux-arm64` need to be built manually on local machines. The generated GitHub release should be edited, with the built binaries uploaded as extra assets (named `workerd-darwin-arm64` and `workerd-linux-arm64` respectively). +This is pretty simple, and completely automatic—every time the compatibility date in [compatibility-date.capnp](src/workerd/io/compatibility-date.capnp) changes, a new release is generated, along with the built binaries for `linux-64` and `darwin-64`. This is governed by the [release.yml](.github/workflows/release.yml) GitHub Action. Since this only generates binaries for `darwin-64` and `linux-64`, binaries for `darwin-arm64` and `linux-arm64` need to be built manually on local machines. These can be generated from a local checkout of `workerd` using the `build-release.sh` script. This must be run on an Apple Silicon machine, and will generate binaries for the latest release on GitHub. The generated GitHub release should be edited, with the built binaries uploaded as extra assets (named `workerd-darwin-arm64` and `workerd-linux-arm64` respectively). ## Publishing `workerd` Since the "Release" stage requires manual upload of binaries, this "Publish" stage requires a manual run of a GitHub Action—the [Publish to NPM](.github/workflows/npm.yml) action. This action has a `workflow_dispatch` trigger that can be activated from within the GitHub UI, and takes two parameters; the patch version, and whether this release is a prerelease. If it _is_ a prerelease, the published NPM version will be tagged with `beta`, and have a version number starting with `0.`. Otherwise, the published NPM version will be tagged `latest`, and have a version number starting with `1.`. diff --git a/build-npm-package.sh b/build-npm-package.sh deleted file mode 100644 index 3914b3afec2..00000000000 --- a/build-npm-package.sh +++ /dev/null @@ -1,71 +0,0 @@ -#! /bin/bash -# TODO(cleanup): Convert to bazel rules/GitHub Actions -set -euo pipefail - -if [ -z "${1-}" ]; then - echo "Please specify a command" - exit 1 -fi -bazel build @capnp-cpp//src/capnp:capnp_tool - -export LATEST_COMPATIBILITY_DATE=$(bazel-bin/external/capnp-cpp/src/capnp/capnp_tool eval src/workerd/io/compatibility-date.capnp supportedCompatibilityDate) -export WORKERD_VERSION=1.$(echo $LATEST_COMPATIBILITY_DATE | tr -d '-' | tr -d '"').0 - -bazel_build() { - bazel build -c opt //src/workerd/server:workerd - mkdir -p $1/bin - node npm/scripts/bump-version.mjs $1/package.json - cp bazel-bin/src/workerd/server/workerd $1/bin/workerd -} - -case $1 in -build-darwin-arm64) - bazel_build npm/workerd-darwin-arm64 - ;; -publish-darwin-arm64) - cd npm/workerd-darwin-arm64 && npm publish - ;; -build-darwin) - bazel_build npm/workerd-darwin-64 - ;; -publish-darwin) - cd npm/workerd-darwin-64 && npm publish - ;; -build-linux-arm64) - bazel_build npm/workerd-linux-arm64 - ;; -publish-linux-arm64) - cd npm/workerd-linux-arm64 && npm publish - ;; -build-linux) - bazel_build npm/workerd-linux-64 - ;; -publish-linux) - cd npm/workerd-linux-64 && npm publish - ;; -build-shim) - node npm/scripts/bump-version.mjs npm/workerd/package.json - mkdir -p npm/workerd/lib - mkdir -p npm/workerd/bin - npx esbuild npm/lib/node-install.ts --outfile=npm/workerd/install.js --bundle --target=node16 --define:LATEST_COMPATIBILITY_DATE=$LATEST_COMPATIBILITY_DATE --platform=node --external:workerd --log-level=warning - npx esbuild npm/lib/node-shim.ts --outfile=npm/workerd/bin/workerd --bundle --target=node16 --define:LATEST_COMPATIBILITY_DATE=$LATEST_COMPATIBILITY_DATE --platform=node --external:workerd --log-level=warning - npx esbuild npm/lib/node-path.ts --outfile=npm/workerd/lib/main.js --bundle --target=node16 --define:LATEST_COMPATIBILITY_DATE=$LATEST_COMPATIBILITY_DATE --platform=node --external:workerd --log-level=warning - node npm/scripts/build-shim-package.mjs - ;; -publish-shim) - cd npm/workerd && npm publish - ;; -clean) - rm -f npm/workerd/install.js - rm -rf npm/workerd-darwin-64/bin - rm -rf npm/workerd-darwin-arm64/bin - rm -rf npm/workerd-linux-64/bin - rm -rf npm/workerd-linux-arm64/bin - rm -rf npm/workerd/bin - rm -rf npm/workerd/lib - ;; -*) - echo "Invalid command" - exit 1 - ;; -esac diff --git a/build-releases.sh b/build-releases.sh new file mode 100755 index 00000000000..a963bacc028 --- /dev/null +++ b/build-releases.sh @@ -0,0 +1,22 @@ +#! /bin/bash +set -euo pipefail + +if [[ $(uname -m) == 'x86_64' ]]; then + echo "This _must_ be run on an Apple Silicon machine, since the macOS ARM build cannot be dockerised due to macOS license restrictions" + exit 1 +fi + +rm -f workerd-darwin-arm64 +rm -f workerd-linux-arm64 + +# Get the tag associated with the latest release, to ensure parity between binaries +TAG_NAME=$(curl -sL https://api.github.com/repos/cloudflare/workerd/releases/latest | jq -r ".tag_name") + +git checkout $TAG_NAME + +# Build macOS binary +pnpm exec bazelisk build --disk_cache=./.bazel-cache -c opt //src/workerd/server:workerd + +cp bazel-bin/src/workerd/server/workerd ./workerd-darwin-arm64 + +docker buildx build --platform linux/arm64 -f Dockerfile.release -t workerd:$TAG_NAME --target=artifact --output type=local,dest=$(pwd) . diff --git a/package.json b/package.json index 24f7097d374..d55f1119040 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "typescript": "~4.7.4" }, "devDependencies": { + "@bazel/bazelisk": "1.12", "@types/debug": "^4.1.7", "@types/node": "^18.7.18", "@types/prettier": "^2.7.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e00637c3ac9..e5d39c67d87 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1,6 +1,7 @@ lockfileVersion: 5.4 specifiers: + '@bazel/bazelisk': '1.12' '@types/debug': ^4.1.7 '@types/node': ^18.7.18 '@types/prettier': ^2.7.1 @@ -22,6 +23,7 @@ dependencies: typescript: 4.7.4 devDependencies: + '@bazel/bazelisk': 1.12.1 '@types/debug': 4.1.7 '@types/node': 18.8.5 '@types/prettier': 2.7.1 @@ -36,6 +38,11 @@ devDependencies: packages: + /@bazel/bazelisk/1.12.1: + resolution: {integrity: sha512-TGCwVeIiVeQUP6yLpxAg8yluFOC+tBQnWw5l8lqwMxKhRtOA+WaH1CJKAXeCBAaS2MxohhkXq44zj/7AM+t2jg==} + hasBin: true + dev: true + /@esbuild/android-arm/0.15.10: resolution: {integrity: sha512-FNONeQPy/ox+5NBkcSbYJxoXj9GWu8gVGJTVmUyoOCKQFDTrHVKgNSzChdNt0I8Aj/iKcsDf2r9BFwv+FSNUXg==} engines: {node: '>=12'}