feat: ci: automate the new release process

.github/workflows/docker.yml

@@ -4,16 +4,21 @@ on:
   push:
     branches:
       - master
-      - release/*
     tags:
       - v*
+  pull_request:
+    branches:
+      - master
+      - release/v*
+      - releases
   schedule:
     - cron: '0 0 * * *'
   workflow_dispatch:
     inputs:
-      ref:
-        description: The GitHub ref (e.g. refs/tags/v1.0.0) to release
+      publish:
+        description: 'Publish the Docker image'
         required: false
+        default: 'false'
 
 defaults:
   run:
@@ -24,7 +29,7 @@ permissions:
 
 jobs:
   docker:
-    name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ (inputs.ref || github.ref) == 'refs/heads/master' || startsWith(inputs.ref || github.ref, 'refs/tags/') }}]
+    name: Docker (${{ matrix.image }} / ${{ matrix.network }}) [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}]
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -40,13 +45,13 @@ jobs:
           - image: lotus
             network: mainnet
     env:
-      PUBLISH: ${{ github.ref == 'refs/heads/master' || startsWith(inputs.ref || github.ref, 'refs/tags/') }}
+      PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}
     steps:
       - id: channel
         env:
-          IS_MASTER: ${{ (inputs.ref || github.ref) == 'refs/heads/master' }}
-          IS_TAG: ${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }}
-          IS_RC: ${{ contains(inputs.ref || github.ref, '-rc') }}
+          IS_MASTER: ${{ github.ref == 'refs/heads/master' }}
+          IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }}
+          IS_RC: ${{ contains(github.ref, '-rc') }}
           IS_SCHEDULED: ${{ github.event_name == 'schedule' }}
         run: |
           channel=''
@@ -67,12 +72,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
-          ref: ${{ inputs.ref || github.ref }}
       - id: git
-        env:
-          REF: ${{ inputs.ref || github.ref }}
         run: |
-          ref="${REF#refs/heads/}"
+          ref="${GITHUB_REF#refs/heads/}"
           ref="${ref#refs/tags/}"
           sha="$(git rev-parse --short HEAD)"
           echo "ref=$ref" | tee -a "$GITHUB_OUTPUT"
@@ -86,7 +88,7 @@ jobs:
           images: filecoin/${{ matrix.image }}
           tags: |
             type=raw,enable=${{ steps.channel.outputs.channel != '' }},value=${{ steps.channel.outputs.channel }}
-            type=raw,enable=${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }},value=${{ steps.git.outputs.ref }}
+            type=raw,enable=${{ startsWith(github.ref, 'refs/tags/') }},value=${{ steps.git.outputs.ref }}
             type=raw,value=${{ steps.git.outputs.sha }}
           flavor: |
             latest=false

.github/workflows/release.yml

@@ -3,16 +3,30 @@ name: Release
 on:
   push:
     branches:
-      - ci/*
-      - release/*
-    tags:
-      - v*
+      - release/v*
+      - releases
+    paths:
+      - build/version.go
+  pull_request:
+    branches:
+      - release/v*
+      - releases
+    paths:
+      - build/version.go
   workflow_dispatch:
     inputs:
-      ref:
-        description: The GitHub ref (e.g. refs/tags/v1.0.0) to release
+      publish:
+        description: 'Publish the release'
         required: false
-
+        default: 'false'
+      draft:
+        description: 'Create a draft release'
+        required: false
+        default: 'true'
+      target_commitish:
+        description: 'The commitish value that determines where the Git tag is created from'
+        required: false
+        default: ''
 defaults:
   run:
     shell: bash
@@ -21,121 +35,184 @@ permissions:
   contents: read
 
 jobs:
+  check:
+    name: Check which projects need to be built
+    runs-on: ubuntu-latest
+    outputs:
+      projects: ${{ steps.projects.outputs.projects }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - id: projects
+        env:
+          TARGET_REF: ${{ github.base_ref || github.ref }}
+        run: |
+          go run cmd/release/main.go --json list-projects | jq -r '.msg' |
+            jq 'map(select(.version | endswith("-dev") | not))' |
+            jq 'map(select(.released | not))' |
+            jq 'map(select(.prerelease != ((env.TARGET_REF | sub("^refs/heads/"; "")) == "releases")))' |
+            jq -c '.' | xargs -I {} -0 echo "projects={}" |
+            tee -a $GITHUB_OUTPUT
   build:
-    name: Build (${{ matrix.os }}/${{ matrix.arch }})
+    needs: [check]
+    if: needs.check.outputs.projects != '[]'
+    name: Build ${{ matrix.project }} (${{ matrix.runner }})
     runs-on: ${{ matrix.runner }}
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - runner: ubuntu-latest
-            os: Linux
-            arch: X64
-          - runner: macos-13
-            os: macOS
-            arch: X64
-          - runner: macos-14
-            os: macOS
-            arch: ARM64
+        project: ${{ fromJSON(needs.check.outputs.projects).*.name }}
+        runner:
+          - ubuntu-latest # Linux X64
+          - macos-13 # MacOs X64
+          - macos-14 # MacOS ARM64
     steps:
-      - env:
-          OS: ${{ matrix.os }}
-          ARCH: ${{ matrix.arch }}
+      - run: echo "Building on $RUNNER_OS/$RUNNER_ARCH"
+      - id: project
+        env:
+          projects: ${{ needs.check.outputs.projects }}
+          name: ${{ matrix.project }}
         run: |
-          if [[ "$OS" != "$RUNNER_OS" || "$ARCH" != "$RUNNER_ARCH" ]]; then
-            echo "::error title=Unexpected Runner::Expected $OS/$ARCH, got $RUNNER_OS/$RUNNER_ARCH"
-            exit 1
-          fi
-      - uses: actions/checkout@v4
-        with:
-          path: actions
+          jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' |
+            xargs -I {} -0 echo "config={}" |
+            tee -a $GITHUB_OUTPUT
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
-          ref: ${{ inputs.ref || github.ref }}
-          path: lotus
-      - uses: ./actions/.github/actions/install-system-dependencies
-      - uses: ./actions/.github/actions/install-go
-        with:
-          working-directory: lotus
+      - uses: ./.github/actions/install-system-dependencies
+      - uses: ./.github/actions/install-go
       - env:
           GITHUB_TOKEN: ${{ github.token }}
-        run: make deps lotus lotus-miner lotus-worker
-        working-directory: lotus
+        run: make deps
+      - if: matrix.project == 'node'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: make lotus
+      - if: matrix.project == 'miner'
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: make lotus-miner lotus-worker
       - if: runner.os == 'macOS'
-        run: otool -hv lotus
-        working-directory: lotus
+        run: if [[ -f lotus ]]; then otool -hv lotus; fi
       - env:
-          INPUTS_REF: ${{ inputs.ref }}
+          LOTUS_VERSION_IGNORE_COMMIT: 1
+          expected: ${{ fromJSON(steps.project.outputs.config).version }}
         run: |
-          export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF}
-          ../actions/scripts/version-check.sh ./lotus
-        working-directory: lotus
+          for bin in lotus lotus-miner lotus-worker; do
+            if [[ -f ./$bin ]]; then
+              chmod +x ./$bin
+              actual=$(./$bin --version | cut -d' ' -f3)
+              if [[ "$actual" != "$expected" ]]; then
+                echo "::error title=Version Mismatch::Expected $expected, got $actual (./$bin)"
+                exit 1
+              fi
+            fi
+          done
       - uses: actions/upload-artifact@v4
         with:
-          name: lotus-${{ matrix.os }}-${{ matrix.arch }}
+          name: lotus-${{ matrix.project }}-${{ runner.os }}-${{ runner.arch }}
           path: |
-            lotus/lotus
-            lotus/lotus-miner
-            lotus/lotus-worker
+            lotus
+            lotus-miner
+            lotus-worker
   release:
-    name: Release [publish=${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }}]
+    needs: [check, build]
+    if: needs.check.outputs.projects != '[]'
+    name: Release ${{ matrix.project }} [publish=${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}]
     permissions:
       # This enables the job to create and/or update GitHub releases
       contents: write
     runs-on: ubuntu-latest
-    needs: [build]
+    strategy:
+      fail-fast: false
+      matrix:
+        project: ${{ fromJSON(needs.check.outputs.projects).*.name }}
     env:
-      PUBLISH: ${{ startsWith(inputs.ref || github.ref, 'refs/tags/') }}
+      PUBLISH: ${{ inputs.publish == 'true' || github.event_name != 'pull_request' }}
+      IS_DRAFT: ${{ inputs.draft == 'true' || github.event_name == 'pull_request' }}
     steps:
-      - uses: actions/checkout@v4
-        with:
-          path: actions
+      - id: project
+        env:
+          projects: ${{ needs.check.outputs.projects }}
+          name: ${{ matrix.project }}
+        run: |
+          jq -nc 'env.projects | fromjson | map(select(.name == env.name)) | .[0]' |
+            xargs -I {} -0 echo "config={}" |
+            tee -a $GITHUB_OUTPUT
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
           fetch-depth: 0
-          path: lotus
-          ref: ${{ inputs.ref || github.ref }}
       - uses: actions/download-artifact@v4
         with:
-          name: lotus-Linux-X64
+          name: lotus-${{ matrix.project }}-Linux-X64
           path: linux_amd64_v1
       - uses: actions/download-artifact@v4
         with:
-          name: lotus-macOS-X64
+          name: lotus-${{ matrix.project }}-macOS-X64
           path: darwin_amd64_v1
       - uses: actions/download-artifact@v4
         with:
-          name: lotus-macOS-ARM64
+          name: lotus-${{ matrix.project }}-macOS-ARM64
           path: darwin_arm64
-      - uses: ./actions/.github/actions/install-go
-        with:
-          working-directory: lotus
+      - uses: ./.github/actions/install-go
       - uses: ipfs/download-ipfs-distribution-action@v1
         with:
           name: kubo
           version: v0.16.0
+          cache: false
+      - name: Install yq
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          gh release --repo mikefarah/yq download v4.44.2 -p yq_linux_amd64
+          sudo mv yq_linux_amd64 /usr/bin/yq
+          sudo chmod +x /usr/bin/yq
+      - if: env.IS_DRAFT == 'false'
+        run: yq -i '.release.draft = false' '.goreleaser.yaml'
+      - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false' && fromJSON(steps.project.outputs.config).latest && !fromJSON(steps.project.outputs.config).prerelease
+        run: yq -i '.brews.[0].skip_upload = false' '.goreleaser.yaml'
+      - if: matrix.project == 'node'
+        env:
+          BREW_INSTALL: |
+            bin.install "lotus"
+          BREW_TEST: |
+            system "#{bin}/lotus --version"
+        run: |
+          yq -i '(.builds[] | select(.id == "lotus")).skip = false' '.goreleaser.yaml'
+          yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml'
+          yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml'
+      - if: matrix.project == 'miner'
+        env:
+          BREW_INSTALL: |
+            bin.install "lotus-miner"
+            bin.install "lotus-worker"
+          BREW_TEST: |
+            system "#{bin}/lotus-miner --version"
+            system "#{bin}/lotus-worker --version"
+        run: |
+          yq -i '(.builds[] | select(.id == "lotus-miner" or .id == "lotus-worker")).skip = false' '.goreleaser.yaml'
+          yq -i '.brews[0].install = strenv(BREW_INSTALL)' '.goreleaser.yaml'
+          yq -i '.brews[0].test = strenv(BREW_TEST)' '.goreleaser.yaml'
       - uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
         with:
           distribution: goreleaser-pro
           version: 2.0.1
-          args: release --clean ${{ env.PUBLISH == 'false' && '--snapshot' || '' }}
-          workdir: lotus
+          args: release --clean --skip=validate ${{ env.PUBLISH == 'false' && '--snapshot' || '' }}
         env:
-          GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_GITUB_TOKEN || github.token || '' }}
+          PROJECT_NAME: ${{ matrix.project == 'node' && 'lotus' || format('lotus-{0}', matrix.project) }}
+          GITHUB_TOKEN: ${{ env.PUBLISH == 'true' && (secrets.GORELEASER_GITHUB_TOKEN || github.token) || '' }}
           GORELEASER_KEY: ${{ env.PUBLISH == 'true' && secrets.GORELEASER_KEY || '' }}
-      - env:
-          INPUTS_REF: ${{ inputs.ref }}
-        run: |
-          export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF}
-          ../actions/scripts/generate-checksums.sh
-        working-directory: lotus
-      - if: env.PUBLISH == 'true'
+          TAG: ${{ fromJSON(steps.project.outputs.config).tag }}
+          VERSION: ${{ fromJSON(steps.project.outputs.config).version }}
+          IS_LATEST: ${{ fromJSON(steps.project.outputs.config).latest }}
+          IS_PRERELEASE: ${{ fromJSON(steps.project.outputs.config).prerelease }}
+          TARGET_COMMITISH: ${{ inputs.target_commitish || (github.event_name == 'pull_request' && github.event.pull_request.base.ref || github.sha) }}
+          HEADER: ''
+      - run: ./scripts/generate-checksums.sh
+      - if: env.PUBLISH == 'true' && env.IS_DRAFT == 'false'
         env:
           GITHUB_TOKEN: ${{ github.token }}
-          INPUTS_REF: ${{ inputs.ref }}
-        run: |
-          export GITHUB_REF=${INPUTS_REF:-$GITHUB_REF}
-          ../actions/scripts/publish-checksums.sh
-        working-directory: lotus
+          TAG: ${{ fromJSON(steps.project.outputs.config).tag }}
+        run: ./scripts/publish-checksums.sh

.gitignore

@@ -53,6 +53,9 @@ build/builtin-actors/v*
 build/builtin-actors/*.car
 
 dist/
+darwin_amd64_v1/
+darwin_arm64/
+linux_amd64_v1/
 
 
 # The following files are checked into git and result