Build, test, and deploy a Heroku slug instead of re-building on Heroku

.github/workflows/ci.yml

@@ -5,9 +5,17 @@ on:
       - master
   pull_request:
   workflow_dispatch:
+defaults:
+  run:
+    # This is the same as GitHub Action's `bash` keyword as of 20 June 2023:
+    # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
+    #
+    # Completely spelling it out here so that GitHub can't change it out from under us
+    # and we don't have to refer to the docs to know the expected behavior.
+    shell: bash --noprofile --norc -eo pipefail {0}
 jobs:
-  test:
-    if: github.repository == 'nextstrain/nextstrain.org'
+  # Lint regardless of build or test status and lint early.
+  lint:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -16,18 +24,59 @@ jobs:
           node-version-file: 'package.json'
       - run: node --version
       - run: npm ci
-      # building the static site (Next.js) will fail if linting fails, so explicitly run linting
-      # first to make the test output clearer
       - run: npm run lint:server
       - run: npm run lint:static-site
-      - run: npm run build
+
+  # Build into Heroku slug so we can deploy the same build that we test.
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: ./scripts/heroku-build
+
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: build
+          path: |
+            build/slug.tar.gz
+            build/slug.tar.gz.sha256sum
+            build/slug.json
+
+  # Run tests in Heroku runtime environment against our built slug.
+  test:
+    if: github.repository == 'nextstrain/nextstrain.org'
+    needs: build
+    runs-on: ubuntu-latest
+    container:
+      image: heroku/heroku:20
+    env:
+      # Override NODE_ENV=production default in production build (slug)
+      NODE_ENV: development
+
+      # Set up Heroku runtime environment.  See bash(1) or
+      # <https://www.gnu.org/software/bash/manual/bash.html#Invoked-non_002dinteractively>.
+      BASH_ENV: .heroku/BASH_ENV
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: build
+          path: build/
+      - run: sha256sum --check build/slug.tar.gz.sha256sum
+      - run: tar --extract --file build/slug.tar.gz --xform 's,^[.]/app/,,'
+      - run: node --version
+
+      # (Re-)install dev deps which got pruned from production build (slug)
+      - run: npm ci
+
       # configure AWS to run dev server necessary for `npm run test:ci`
       - uses: aws-actions/configure-aws-credentials@v4
         with:
           aws-region: ${{ vars.AWS_DEFAULT_REGION }}
           aws-access-key-id: ${{ secrets.DEV_SERVER_AWS_ACCESS_KEY_ID }}
           aws-secret-access-key: ${{ secrets.DEV_SERVER_AWS_SECRET_ACCESS_KEY }}
-      - run: aws sts get-caller-identity
+
       - run: npm run test:ci
         env:
           # Tests make GitHub API requests which are rate limited.
@@ -38,25 +87,19 @@ jobs:
       - if: always()
         uses: actions/upload-artifact@v4
         with:
-          name: logs
+          name: test-logs
           path: test/server.log
 
-  # XXX TODO: It'd be nice to avoid the rebuild on Heroku and instead deploy
-  # the artifacts (source code + generated files + node_modules/) already built
-  # above.  This would dramatically reduce deploy times and move us closer to
-  # "deploy what you tested", but it may also run into platform compatibility
-  # issues given CI is building on a different platform (arch + OS + sys libs)
-  # than Heroku's dynos and some deps are compiled.  But should try it and see!
-  # Or do our build above inside a Heroku buildpack…
-  #   -trs, 2 May 2022
   deploy:
     if: |2
          github.repository == 'nextstrain/nextstrain.org'
       && github.event_name == 'push'
       && github.ref == 'refs/heads/master'
 
-    # Wait for "test" job above to pass.
-    needs: test
+    # Wait for "build" and "test" jobs above to pass.
+    needs:
+      - build
+      - test
 
     # Only one "deploy" job at a time.
     concurrency: deploy
@@ -78,61 +121,51 @@ jobs:
     env:
       HEROKU_APP: nextstrain-canary
     steps:
+      - uses: actions/download-artifact@v4
+        with:
+          name: build
+          path: build/
+
       - name: Login to Heroku
         run: echo "machine api.heroku.com login $HEROKU_USER password $HEROKU_TOKEN" >> ~/.netrc
         env:
           HEROKU_USER: "${{ secrets.HEROKU_USER }}"
           HEROKU_TOKEN: "${{ secrets.HEROKU_TOKEN }}"
 
-      - name: Define Heroku build source
+      - name: Upload slug
         run: |
-          jq --null-input '{
-            "source_blob": {
-              "url": "https://github.com/\(env.GITHUB_REPOSITORY)/archive/\(env.GITHUB_SHA).tar.gz",
-              "version": env.GITHUB_SHA
-            }
-          }' | tee build-source.json
-
-      - name: Start Heroku build
-        run: |
-          curl https://api.heroku.com/apps/"$HEROKU_APP"/builds \
-            --fail --silent --show-error --location --netrc \
-            --data-binary @build-source.json \
+          # <https://devcenter.heroku.com/articles/platform-api-reference#slug-create>
+          curl https://api.heroku.com/apps/"$HEROKU_APP"/slugs \
+            --data-binary @build/slug.json \
             --header 'Content-Type: application/json' \
             --header 'Accept: application/vnd.heroku+json; version=3' \
-              | tee build.json
+            --fail --silent --show-error --location --netrc \
+              | tee slug.json
 
-      - name: Monitor Heroku build
-        run: |
-          curl "$(jq -r .output_stream_url build.json)" \
-            --fail --silent --show-error --location
+          curl "$(jq -r .blob.url slug.json)" \
+            --request "$(jq -r .blob.method slug.json | tr a-z A-Z)" \
+            --header "Content-Type:" \
+            --data-binary @build/slug.tar.gz \
+            --fail --location --netrc \
+              | cat
 
-      - name: Check Heroku build
+      - name: Release slug
         run: |
-          id="$(jq -r .id build.json)"
-          SECONDS=0
-          maxwait=60
-
-          while [[ $SECONDS -lt $maxwait ]]; do
-            curl https://api.heroku.com/apps/"$HEROKU_APP"/builds/"$id" \
-              --fail --silent --show-error --location --netrc \
-              --header 'Accept: application/vnd.heroku+json; version=3' \
-                | tee build.json
-
-            status="$(jq -r .status build.json)"
-
-            if [[ "$status" == succeeded ]]; then
-              break
-            else
-              echo "build status is $status (not succeeded); will check again" >&2
-              sleep 5
-            fi
-          done
-
-          if [[ "$status" != succeeded ]]; then
-            echo "build status is $status (not succeeded) after waiting $SECONDS seconds" >&2
-            exit 1
-          fi
+          # <https://devcenter.heroku.com/articles/platform-api-reference#release-create>
+          curl https://api.heroku.com/apps/"$HEROKU_APP"/releases \
+            --data-binary @<(jq '{slug: .id}' slug.json) \
+            --header 'Content-Type: application/json' \
+            --header 'Accept: application/vnd.heroku+json; version=3' \
+            --fail --silent --show-error --location --netrc \
+              | tee release.json
+
+      - if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: deploy
+          path: |
+            slug.json
+            release.json
 
       - if: always()
         name: Logout of Heroku

.gitignore

@@ -1,6 +1,9 @@
 # environment variables
 /environment*
 
+# Heroku build
+/build/
+
 # auspice build
 /auspice-client/index.html
 /auspice-client/dist/
@@ -46,4 +49,4 @@ terraform.tfstate*
 
 # nextjs
 static-site/next-env.d.ts
-static-site/.next
+static-site/.next

docs/infrastructure.md

@@ -142,7 +142,7 @@ To recreate an inactivated app, or create one for a PR from a fork, you can use
 
 ### Rolling back deployments
 
-Normal Heroku deployments, which require our GitHub Actions CI tests to pass and are subsequently built on Heroku, can take upwards of 10 minutes.
+Normal Heroku deployments, performed by our GitHub Actions CI workflow, take about 6 minutes.
 Heroku allows us to immediately return to a previous version using rollbacks.
 Rollbacks can be performed via the Heroku dashboard or with `heroku rollback --app=nextstrain-server vX`, where _X_ is the version number (available via `heroku releases --app=nextstrain-server`).
 

scripts/heroku-build

@@ -0,0 +1,112 @@
+#!/bin/bash
+# Build a Heroku app slug using the Heroku (classic) buildpack interface.
+# <https://devcenter.heroku.com/articles/buildpack-api>
+#
+set -euo pipefail
+
+cd "$(dirname "$0")/.."
+
+echo "---> Hello from $0"
+echo "     Source directory is $PWD"
+echo "     Build  directory is $PWD/build"
+echo
+echo "---> Removing any previous build/app/"
+rm -rf build/app/
+
+echo
+echo "---> Ensuring build dirs exist"
+mkdir -vp build/{app,cache,env}
+
+echo
+echo "---> Copying source to build/app/"
+# Don't use `git archive` because it excludes uncommitted (but unignored) files
+# in the working tree.  Pass thru tar so that it handles directory creation for
+# us.
+git ls-files --cached --others --exclude-standard -z \
+  | tar --create  --file - --null --files-from - \
+  | tar --extract --file - --verbose -C build/app/
+
+echo
+echo "---> Setting buildpack env vars from source"
+export SOURCE_VERSION SOURCE_DESCRIPTION
+echo "     SOURCE_VERSION=${SOURCE_VERSION:=$(git describe --always --dirty)}"
+echo "     SOURCE_DESCRIPTION=${SOURCE_DESCRIPTION:=$(git log -1 --format=%s)}"
+
+if [[ ! -d build/pack ]]; then
+  echo
+  echo "---> Cloning buildpack into build/pack/"
+  git clone --depth 1 https://github.com/heroku/heroku-buildpack-nodejs build/pack
+else
+  echo
+  echo "---> Using buildpack in build/pack/"
+fi
+
+echo
+echo "---> Buildpack version is $(git -C build/pack describe --always --dirty)"
+
+echo
+echo "---> Running buildpack"
+docker run \
+  --rm --interactive $(tty --quiet && echo --tty) \
+  --user $(id -u):$(id -g) \
+  --volume "$(realpath build/app)":/build/app:rw \
+  --volume "$(realpath build/pack)":/build/pack:ro \
+  --volume "$(realpath build/cache)":/build/cache:rw \
+  --volume "$(realpath build/env)":/build/env:ro \
+  --env STACK=heroku-20 \
+  --env SOURCE_VERSION \
+  --env SOURCE_DESCRIPTION \
+  --env NODE_MODULES_CACHE=false \
+  heroku/heroku:20-build bash -c '
+       /build/pack/bin/detect  /build/app \
+    && /build/pack/bin/compile /build/app /build/cache /build/env
+  '
+
+# The remaining steps are part of the unpublished (but somewhat documented)
+# "slug compiler"¹ for classic Heroku buildpacks.
+#
+# The Procfile conversion is a published part of Heroku's upcoming "Cloud
+# Native" buildpacks, though, so nab the per-line regex from there.²
+#
+# ¹ <https://devcenter.heroku.com/articles/slug-compiler>
+# ² <https://github.com/heroku/buildpacks-procfile/blob/df64135f/src/procfile.rs#L42-L44>
+echo
+echo "---> Converting Procfile to JSON"
+jq --slurp --raw-input '
+    split("\n")
+  | map(capture("^[[:space:]]*(?<key>[a-zA-Z0-9_-]+):?\\s+(?<value>.*)[[:space:]]*"))
+  | from_entries
+' < build/app/Procfile | tee build/app/Procfile.json
+
+echo
+echo "---> Writing build/app/.heroku/BASH_ENV for use with BASH_ENV"
+tee build/app/.heroku/BASH_ENV <<<'
+  for f in "$(dirname "$BASH_SOURCE")"/../.profile.d/*; do
+      # On Heroku dynos, the slug is at /app and HOME=/app, but that is not
+      # true elsewhere and HOME often points somewhere completely separate.
+      # Make the .profile.d/ scripts portable when using this BASH_ENV script.
+      HOME="$(realpath "$(dirname "$BASH_SOURCE")"/..)" source "$f"
+  done
+'
+
+# <https://devcenter.heroku.com/articles/platform-api-deploying-slugs>
+echo
+echo "---> Packing build/app/ into build/slug.tar.gz"
+tar --create --gzip --file build/slug.tar.gz -C build/ ./app/
+sha256sum build/slug.tar.gz > build/slug.tar.gz.sha256sum
+du --human-readable --si --summarize build/app/ build/slug.tar.gz
+
+# <https://devcenter.heroku.com/articles/platform-api-reference#slug-create>
+echo
+echo "---> Generating build/slug.json metadata for slug creation"
+jq \
+  --null-input \
+  --argjson procfile "$(<build/app/Procfile.json)" \
+  --arg checksum "$(awk '{print "SHA256:" $1}' build/slug.tar.gz.sha256sum)" \
+  '{
+    "stack": "heroku-20",
+    "commit": $ENV.SOURCE_VERSION,
+    "commit_description": $ENV.SOURCE_DESCRIPTION,
+    "process_types": $procfile,
+    "checksum": $checksum,
+  }' | tee build/slug.json