From 37475a03bc1dc87b7723fba9af229f43a3f73219 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Sat, 30 Nov 2024 18:09:26 +0000 Subject: [PATCH] feat(builds): improve build speeds and use free arm64 runners Improves a few things: - Use the free macos-14 runners for arm64 builds. They are not as fast the macos-13-xlarge runners used before, but they are free for public repositories, meaning we can start doing nightly arm64 builds. - Use different nix install and cache actions which are faster, and uses a single action cache key, which avoids cache rate limit errors which was slowing down cache create/restore times. - Generally refactor and tidy up various bits and pieces in workflows. --- .github/workflows/_build.yml | 123 ++++++++++++++------------------- .github/workflows/_prepare.yml | 66 +++++++++++++----- .github/workflows/_release.yml | 14 ++-- .github/workflows/build.yml | 41 +++++------ 4 files changed, 123 insertions(+), 121 deletions(-) diff --git a/.github/workflows/_build.yml b/.github/workflows/_build.yml index c30fb25..f53beb7 100644 --- a/.github/workflows/_build.yml +++ b/.github/workflows/_build.yml @@ -4,24 +4,24 @@ name: _build on: workflow_call: inputs: - should_run: - description: Whether or not to run the build job - type: boolean - default: false - artifact_prefix: - description: Artifact prefix + builder_ref: + description: Git ref of build-emacs-for-macos to use type: string - required: false + required: true os: description: GitHub Actions runner OS type: string required: false - default: "macos-12" + default: "macos-13" build_os: description: Target OS to build for type: string required: false - default: "macos-12" + default: "macos-13" + artifact_prefix: + description: Artifact prefix for build_os + type: string + required: false git_ref: description: Git ref to build type: string @@ -65,43 +65,13 @@ on: AC_SIGN_IDENTITY: description: Apple Connect Signing Identify required: true - TAP_REPO_TOKEN: - description: Homebrew Tap Token - required: true outputs: package_created: description: "Whether or not a package was created" value: ${{ jobs.package.result == 'success' }} jobs: - prepare: - runs-on: ${{ inputs.os }} - outputs: - builder_sha: ${{ steps.builder_sha.outputs.sha }} - emacs_sha_override: ${{ steps.emacs_sha.outputs.sha }} - test_plan_args: ${{ steps.test_plan_args.outputs.args }} - steps: - - name: Download emacs-builder git SHA artifact - uses: actions/download-artifact@v4 - with: - name: emacs-builder-git-sha - path: ./ - - name: Store builder Git SHA - id: builder_sha - run: >- - echo "sha=$(cat emacs-builder-git-sha.txt)" >> $GITHUB_OUTPUT - - name: Prepare plan test args - id: test_plan_args - if: ${{ inputs.test_build_name != '' }} - run: >- - echo "args=--test-build '${{ inputs.test_build_name }}' --test-release-type '${{ inputs.test_release_type }}'" >> $GITHUB_OUTPUT - - name: Set git SHA override - id: emacs_sha - if: ${{ inputs.git_sha != '' }} - run: >- - echo "sha=--sha '${{ inputs.git_sha }}'" >> $GITHUB_OUTPUT plan: - needs: [prepare] runs-on: ${{ inputs.build_os }} outputs: check: ${{ steps.check.outputs.result }} @@ -110,32 +80,37 @@ jobs: uses: actions/checkout@v4 with: repository: jimeh/build-emacs-for-macos - ref: ${{ needs.prepare.outputs.builder_sha }} - - uses: actions/setup-go@v5 - if: ${{ inputs.os != inputs.build_os }} - with: - go-version: "1.23" - - name: Build emacs-builder tool - if: ${{ inputs.os != inputs.build_os }} - run: make build + ref: ${{ inputs.builder_ref }} - name: Download pre-built emacs-builder artifact - if: ${{ inputs.os == inputs.build_os }} uses: actions/download-artifact@v4 with: - name: emacs-builder + name: emacs-builder-${{ runner.arch }} path: bin - name: Ensure emacs-builder is executable - if: ${{ inputs.os == inputs.build_os }} run: chmod +x bin/emacs-builder - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: nixbuild/nix-quick-install-action@v29 + - uses: nix-community/cache-nix-action@v5 + with: + primary-key: nix-${{ runner.arch }}-${{ hashFiles('**/flake.*') }} + - name: Install dependencies + run: nix develop --command nix flake metadata + - name: Prepare plan test args + id: test_plan_args + if: inputs.test_build_name != '' + run: >- + echo "args=--test-build '${{ inputs.test_build_name }}' --test-release-type '${{ inputs.test_release_type }}'" >> "$GITHUB_OUTPUT" + - name: Set git SHA override + id: emacs_sha + if: inputs.git_sha != '' + run: >- + echo "sha=--sha '${{ inputs.git_sha }}'" >> "$GITHUB_OUTPUT" - name: Plan build run: >- nix develop --command bin/emacs-builder -l debug plan --output build-plan.yml --output-dir '${{ github.workspace }}/builds' - ${{ needs.prepare.outputs.test_plan_args }} - ${{ needs.prepare.outputs.emacs_sha_override }} + ${{ steps.test_plan_args.outputs.args }} + ${{ steps.emacs_sha.outputs.sha }} '${{ inputs.git_ref }}' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -151,35 +126,39 @@ jobs: id: check continue-on-error: true run: | - echo "result=$((bin/emacs-builder -l debug release --plan build-plan.yml check && echo 'ok') || echo 'fail')" >> $GITHUB_OUTPUT + echo "result=$((bin/emacs-builder -l debug release --plan build-plan.yml check && echo 'ok') || echo 'fail')" >> "$GITHUB_OUTPUT" env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: echo 'Planned release already seems to exist.' - if: ${{ steps.check.outputs.result == 'ok' }} + if: steps.check.outputs.result == 'ok' build: runs-on: ${{ inputs.build_os }} - needs: [prepare, plan] + needs: [plan] # Only run if check for existing release and asset failed. - if: ${{ needs.plan.outputs.check == 'fail' }} + if: needs.plan.outputs.check == 'fail' steps: - name: Checkout build-emacs-for-macos repo uses: actions/checkout@v4 with: repository: jimeh/build-emacs-for-macos - ref: ${{ needs.prepare.outputs.builder_sha }} + ref: ${{ inputs.builder_ref }} path: builder - - uses: DeterminateSystems/nix-installer-action@main - - uses: DeterminateSystems/magic-nix-cache-action@main + - uses: nixbuild/nix-quick-install-action@v29 + - uses: nix-community/cache-nix-action@v5 + with: + primary-key: nix-${{ runner.arch }}-${{ hashFiles('**/flake.*') }} - name: Download build-plan artifact uses: actions/download-artifact@v4 with: name: ${{ inputs.artifact_prefix }}build-plan path: ./builder/ - name: Install dependencies + run: nix develop --command nix flake metadata + working-directory: builder + - name: Install Ruby dependencies run: >- - nix develop - --command make bootstrap-ruby + nix develop --command make bootstrap-ruby working-directory: builder env: BUNDLE_WITHOUT: "development" @@ -209,21 +188,21 @@ jobs: package: runs-on: ${{ inputs.os }} - needs: [prepare, plan, build] + needs: [plan, build] steps: - - uses: actions/setup-python@v5 - with: - python-version: "3.11" - - name: Install dmgbuild - run: | - $(command -v pip3 || command -v pip) install --upgrade dmgbuild - name: Download pre-built emacs-builder artifact uses: actions/download-artifact@v4 with: - name: emacs-builder + name: emacs-builder-${{ runner.arch }} path: bin - name: Ensure emacs-builder is executable run: chmod +x bin/emacs-builder + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install dmgbuild + run: | + $(command -v pip3 || command -v pip) install --upgrade dmgbuild - name: Download build-plan artifact uses: actions/download-artifact@v4 with: @@ -278,6 +257,6 @@ jobs: builds/*.sha* if-no-files-found: error - name: Clean up keychain used for signing certificate - if: ${{ always() }} + if: always() run: | security delete-keychain "$RUNNER_TEMP/app-signing.keychain-db" diff --git a/.github/workflows/_prepare.yml b/.github/workflows/_prepare.yml index 000fdca..db65bfa 100644 --- a/.github/workflows/_prepare.yml +++ b/.github/workflows/_prepare.yml @@ -3,24 +3,21 @@ name: _prepare on: workflow_call: inputs: - os: - description: GitHub Actions runner OS - type: string - required: false - default: "macos-12" builder_ref: description: Git ref to checkout of build-emacs-for-macos required: false type: string - default: "v0.6.50" - secrets: - TAP_REPO_TOKEN: - description: Personal Access Token for Homebrew Tap repo - required: true + default: "v0.6.52" + outputs: + builder_sha: + description: Git SHA of build-emacs-for-macos at builder_ref + value: ${{ jobs.builder-sha.outputs.sha }} jobs: - emacs-builder: - runs-on: ${{ inputs.os }} + builder-sha: + runs-on: "macos-13" + outputs: + sha: ${{ steps.builder_sha.outputs.sha }} steps: - name: Checkout build-emacs-for-macos repo uses: actions/checkout@v4 @@ -28,22 +25,57 @@ jobs: repository: jimeh/build-emacs-for-macos ref: ${{ inputs.builder_ref }} - name: Store builder Git SHA + id: builder_sha run: | - git rev-parse HEAD > emacs-builder-git-sha.txt + BUILDER_SHA="$(git rev-parse HEAD)" + echo "$BUILDER_SHA" > build-emacs-for-macos-git-sha.txt + echo "sha=$BUILDER_SHA" >> $GITHUB_OUTPUT + echo "Builder ref ${{ inputs.builder_ref }} resolved to" \ + "commit SHA: $BUILDER_SHA" - name: Upload builder git SHA artifact uses: actions/upload-artifact@v4 with: - name: emacs-builder-git-sha - path: emacs-builder-git-sha.txt + name: build-emacs-for-macos-git-sha + path: build-emacs-for-macos-git-sha.txt if-no-files-found: error - - uses: actions/setup-go@v5 + + emacs-builder: + needs: [builder-sha] + strategy: + matrix: + os: + - macos-13 # Only macos-13 and earlier are x86_64. + - macos-14 # Only macos-14 and later are ARM64. + runs-on: ${{ matrix.os }} + steps: + - name: Cache emacs-builder (${{ runner.arch }}) + id: cache + uses: actions/cache@v4 + with: + path: bin/emacs-builder + key: emacs-builder-${{ runner.arch }}-${{ needs.builder-sha.outputs.sha }}-bin + - name: Checkout build-emacs-for-macos repo + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/checkout@v4 + with: + repository: jimeh/build-emacs-for-macos + ref: ${{ inputs.builder_ref }} + fetch-depth: 0 + - name: Setup Go + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/setup-go@v5 with: go-version: "1.23" - name: Build emacs-builder tool + if: steps.cache.outputs.cache-hit != 'true' run: make build + - name: Ensure emacs-builder is executable + if: steps.cache.outputs.cache-hit != 'true' + run: chmod +x bin/emacs-builder + - run: bin/emacs-builder --version - name: Upload emacs-builder artifact uses: actions/upload-artifact@v4 with: - name: emacs-builder + name: emacs-builder-${{ runner.arch }} path: bin/emacs-builder if-no-files-found: error diff --git a/.github/workflows/_release.yml b/.github/workflows/_release.yml index 853e0cf..73ff55d 100644 --- a/.github/workflows/_release.yml +++ b/.github/workflows/_release.yml @@ -7,6 +7,10 @@ concurrency: on: workflow_call: inputs: + builder_ref: + description: Git ref of build-emacs-for-macos to use + type: string + required: true os: description: GitHub Actions runner OS type: string @@ -20,10 +24,6 @@ on: description: Name of artifact containing a *.dmg files to release type: string required: true - secrets: - TAP_REPO_TOKEN: - description: Personal Access Token for Homebrew Tap repo - required: true jobs: github: @@ -32,7 +32,7 @@ jobs: - name: Download pre-built emacs-builder artifact uses: actions/download-artifact@v4 with: - name: emacs-builder + name: emacs-builder-${{ runner.arch }} path: bin - name: Ensure emacs-builder is executable run: chmod +x bin/emacs-builder @@ -49,11 +49,11 @@ jobs: name: ${{ inputs.dmg_artifact }} path: builds - name: Publish disk images to a GitHub Release - if: ${{ steps.dmg.outputs.result != 'fail' }} + if: steps.dmg.outputs.result != 'fail' run: >- bin/emacs-builder -l debug release --plan build-plan.yml publish $(find builds -name '*.dmg' -or -name '*.sha256') env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: echo 'No DMG artifact available, was there a new commit to build?' - if: ${{ steps.dmg.outputs.result == 'fail' }} + if: steps.dmg.outputs.result == 'fail' diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 29ab9a8..4499d47 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -13,15 +13,11 @@ on: builder_ref: description: "Git ref to checkout of build-emacs-for-macos" required: true - default: "v0.6.50" + default: "v0.6.52" builder_args: description: Custom arguments passed to build script required: false default: "" - os: - description: 'Runner OS ("macos-13", "macos-14", or "macos-latest")' - required: true - default: "macos-13" test_build_name: description: "Test build name" required: false @@ -38,7 +34,7 @@ on: arm64: description: "Build arm64 version of Emacs" required: false - default: false + default: true type: boolean jobs: @@ -46,10 +42,7 @@ jobs: name: Prepare uses: ./.github/workflows/_prepare.yml with: - os: ${{ github.event.inputs.os }} builder_ref: ${{ github.event.inputs.builder_ref }} - secrets: - TAP_REPO_TOKEN: ${{ secrets.TAP_REPO_TOKEN }} # ---------------------------------------------------------------------------- # Build x86_64 version of Emacs @@ -57,12 +50,13 @@ jobs: build_x86_64: name: Build (x86_64) - if: ${{ github.event.inputs.x86_64 == 'true' }} + if: github.event.inputs.x86_64 == 'true' uses: ./.github/workflows/_build.yml needs: [prepare] with: - os: ${{ github.event.inputs.os }} - build_os: "macos-13" + builder_ref: ${{ needs.prepare.outputs.builder_sha }} + os: "macos-13" + build_os: "macos-13" # Only macos-13 and earlier are x86_64. artifact_prefix: "x86_64-" git_ref: ${{ github.event.inputs.git_ref }} git_sha: ${{ github.event.inputs.git_sha }} @@ -77,7 +71,6 @@ jobs: AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_PROVIDER: ${{ secrets.AC_PROVIDER }} AC_SIGN_IDENTITY: ${{ secrets.AC_SIGN_IDENTITY }} - TAP_REPO_TOKEN: ${{ secrets.TAP_REPO_TOKEN }} release_x86_64: name: Release (x86_64) @@ -85,18 +78,17 @@ jobs: # Depend on both build_x86_64 and build_arm64, but only run if build_x86_64 # was successful and a package was created. This ensure wait for all builds # to complete before running any release jobs. - needs: [build_x86_64, build_arm64] + needs: [prepare, build_x86_64, build_arm64] if: | always() && needs.build_x86_64.result == 'success' && needs.build_x86_64.outputs.package_created && needs.build_arm64.result != 'failure' with: - os: ${{ github.event.inputs.os }} + builder_ref: ${{ needs.prepare.outputs.builder_sha }} + os: "macos-13" # Only macos-13 and earlier are x86_64. plan_artifact: x86_64-build-plan dmg_artifact: x86_64-dmg - secrets: - TAP_REPO_TOKEN: ${{ secrets.TAP_REPO_TOKEN }} # ---------------------------------------------------------------------------- # Build arm64 version of Emacs @@ -104,12 +96,13 @@ jobs: build_arm64: name: Build (arm64) - if: ${{ github.event.inputs.arm64 == 'true' }} + if: github.event.inputs.arm64 == 'true' uses: ./.github/workflows/_build.yml needs: [prepare] with: - os: ${{ github.event.inputs.os }} - build_os: "macos-13-xlarge" # Only macos-13-xlarge has arm64 support. + builder_ref: ${{ needs.prepare.outputs.builder_sha }} + os: "macos-14" + build_os: "macos-14" # Only macos-14 and later are ARM64. artifact_prefix: "arm64-" git_ref: ${{ github.event.inputs.git_ref }} git_sha: ${{ github.event.inputs.git_sha }} @@ -124,7 +117,6 @@ jobs: AC_PASSWORD: ${{ secrets.AC_PASSWORD }} AC_PROVIDER: ${{ secrets.AC_PROVIDER }} AC_SIGN_IDENTITY: ${{ secrets.AC_SIGN_IDENTITY }} - TAP_REPO_TOKEN: ${{ secrets.TAP_REPO_TOKEN }} release_arm64: name: Release (arm64) @@ -132,18 +124,17 @@ jobs: # Depend on both build_arm64 and build_x86_64, but only run if build_arm64 # was successful and a package was created. This ensure wait for all builds # to complete before running any release jobs. - needs: [build_arm64, build_x86_64] + needs: [prepare, build_arm64, build_x86_64] if: | always() && needs.build_arm64.result == 'success' && needs.build_arm64.outputs.package_created && needs.build_x86_64.result != 'failure' with: - os: ${{ github.event.inputs.os }} + builder_ref: ${{ needs.prepare.outputs.builder_sha }} + os: "macos-14" # Only macos-14 and later are ARM64. plan_artifact: arm64-build-plan dmg_artifact: arm64-dmg - secrets: - TAP_REPO_TOKEN: ${{ secrets.TAP_REPO_TOKEN }} # ---------------------------------------------------------------------------- # Trigger update casks workflow in homebrew tap