Skip to content

Commit

Permalink
[ADP-3400] Automate part of the release process as CI steps (#4749)
Browse files Browse the repository at this point in the history
This PR remove some manual steps from the release process.
In particular

- create release tag
- create release
- upload artifacts
- upload the titles and links of the PRs since last release, for further
human processing

Because the release process is rare, this PR introduce a running draft
release called nightly, which runs the automated steps on a running tag
that gets rewritten every day.

More work should be done to
- collect automatically the dockerhub image link.
- collect the bump.sh to the current change

ADP-3400
  • Loading branch information
paolino authored Aug 27, 2024
2 parents e2ddfa9 + 1b8b503 commit 5ba3cbc
Show file tree
Hide file tree
Showing 12 changed files with 286 additions and 93 deletions.
21 changes: 10 additions & 11 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
Expand Up @@ -510,11 +510,12 @@ steps:
if: |
build.branch !~ /^gh-readonly-queue\/master/
&& build.branch != "master"
&& ( build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "TRUE" )
&& build.env("RELEASE_CANDIDATE") == null
key: block-macos

- label: Build Integration Tests (macOS, arm64)
key: macos-arm64-tests-build-integration
if: build.env("TEST_RC") == null || build.env("TEST_RC") == "FALSE"
depends_on: block-macos
command: nix build -L .#packages.aarch64-darwin.integration-exe
agents:
Expand Down Expand Up @@ -610,10 +611,8 @@ steps:
if: |
build.branch !~ /^gh-readonly-queue\/master/
&& build.branch != "master"
&& (build.env("RELEASE_CANDIDATE") == null
|| build.env("TEST_RC") == "TRUE"
)
&& build.branch != "rc-latest"
&& build.env("RELEASE_CANDIDATE") == null
key: trigger-build-windows-artifacts

- label: Build Package (windows)
Expand All @@ -627,6 +626,7 @@ steps:

- label: Build Testing Bundle (windows)
key: windows-testing-bundle
if: build.env("TEST_RC") == null || build.env("TEST_RC") == "FALSE"
depends_on:
- trigger-build-windows-artifacts
command: nix build -o result/windows-tests .#ci.artifacts.win64.tests
Expand All @@ -635,7 +635,7 @@ steps:
system: ${linux}

- label: Tag as Release Candidate (windows)
if: build.env("RELEASE_CANDIDATE") != null
if: build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "FALSE"
depends_on:
- windows-package
- windows-testing-bundle
Expand All @@ -650,21 +650,20 @@ steps:
- linux-artifacts
steps:

- block: Docker Build
depends_on: []
if: build.env("TEST_RC") == "TRUE"
key: docker-build-block

- label: Build Docker Image
key: docker-build
depends_on: docker-build-block
commands:
./scripts/buildkite/main/docker-build.sh
artifact_paths:
- ./artifacts/*.tgz
agents:
system: x86_64-linux

- block: Docker Build
depends_on: []
if: build.env("TEST_RC") == "TRUE"
key: docker-build-block

- label: Push Docker Image
depends_on:
- docker-build
Expand Down
90 changes: 80 additions & 10 deletions .buildkite/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -24,22 +24,63 @@ steps:
commands:
- ./scripts/buildkite/release/generate-trigger.sh | buildkite-agent pipeline upload

- label: Push swagger daily
depends_on: main-pipeline-build
- label: Collect changes
key: collect-changes
depends_on: add-release-commits
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-bump.sh
artifacts:
- artifacts/api-diffs.md
env:
RELEASE: false
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/make_changelog.sh
agents:
system: x86_64-linux
artifacts:
- artifacts/changes.md

- group : Nightly
key: nightly
depends_on: main-pipeline-build
steps:
- label: Push swagger nightly
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-to-bump.sh
artifacts:
- artifacts/api-diffs.md
env:
RELEASE: false
agents:
system: x86_64-linux

- label: Push nightly release tag
key: push-nightly-tag
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-tag.sh
agents:
system: x86_64-linux
env:
RELEASE: false

- label: Push nightly release
depends_on: push-nightly-tag
key: push-nightly-release
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-release.sh
agents:
system: x86_64-linux
env:
RELEASE: false

- label: Push nightly release artifacts
depends_on: push-nightly-release
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-artifacts.sh
agents:
system: x86_64-linux
env:
RELEASE: false

- group: Release
if: build.branch =~ /^release-candidate/
depends_on: nightly
if: build.branch == "master"
steps:
- block: Create a release
depends_on: main-pipeline-build
key: create-release

- label: Push swagger release
Expand All @@ -51,4 +92,33 @@ steps:
env:
RELEASE: true
agents:
system: x86_64-linux
system: x86_64-linux

- label: Push release tag
key: push-release-tag
depends_on: create-release
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-tag.sh
agents:
system: x86_64-linux
env:
RELEASE: true

- label: Push release
depends_on: push-release-tag
key: push-release
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-release.sh
agents:
system: x86_64-linux
env:
RELEASE: true

- label: Push release artifacts
depends_on: push-release
commands:
- nix develop path:$RELEASE_SCRIPTS_DIR -c $RELEASE_SCRIPTS_DIR/push-artifacts.sh
agents:
system: x86_64-linux
env:
RELEASE: true
2 changes: 2 additions & 0 deletions scripts/buildkite/release/flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@
pkgs.gnutar
pkgs.gnupg
pkgs.nodejs_22
pkgs.gh
pkgs.gettext
];
shellHook = ''
# use this hook to set up additional environment variables
Expand Down
12 changes: 10 additions & 2 deletions scripts/buildkite/release/generate-trigger.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash
#! /usr/bin/env bash

set -euo pipefail

Expand All @@ -7,6 +7,13 @@ VERSION=$(buildkite-agent meta-data get "release-version")
BRANCH=$(buildkite-agent meta-data get "release-candidate-branch")
CABAL=$(buildkite-agent meta-data get "release-cabal-version")
TEST_RC=$(buildkite-agent meta-data get "test-rc")
BASE_BUILD=$(buildkite-agent meta-data get "base-build")

if [ "$TEST_RC" == "TRUE" ]; then
title="Test Release Candidate of $VERSION"
else
title="Release Candidate of $VERSION"
fi

cat << YAML
steps:
Expand All @@ -17,7 +24,7 @@ steps:
build:
commit: $COMMIT
branch: $BRANCH
message: Release Candidate of $VERSION
message: $title
env:
RELEASE_CANDIDATE: "$VERSION"
TEST_RC: "$TEST_RC"
Expand All @@ -26,5 +33,6 @@ steps:
release-candidate-commit: "$COMMIT"
release-candidate-branch: "$BRANCH"
release-cabal-version: "$CABAL"
triggered-by: "$BASE_BUILD"
YAML
29 changes: 29 additions & 0 deletions scripts/buildkite/release/make_changelog.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
#!/usr/bin/env bash

set -euo pipefail

since_date="$(buildkite-agent meta-data get last-release-date)"

fetch_prs() {
curl --silent \
-H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/search/issues?q=repo:cardano-foundation/cardano-wallet+is:pr+is:merged+merged:%3E$since_date"
}

prs=$(fetch_prs | jq '.items | map({date:.pull_request.merged_at | split("T")[0], number:.number,title:.title,labels:.labels| map(.name),author:.user.login })')

items=$(jq 'map("\(.date), #\(.number), \(.author), \(.labels | join(", ")), \(.title)")' <<< "$prs")
item_max=$(jq 'length - 1' <<< "$items")

mkdir -p artifacts

for i in $(seq 0 "$item_max"); do
item=$(jq -r ".[$i]" <<< "$items")
echo "$item" \
| sed -E 's,\[(ADP\-[0-9]+)\],[\1](https://cardanofoundation.atlassian.net/browse/\1),g' \
| sed -E 's,\[ADP-x+\],,g' \
| sed -E 's,#([0-9]+),[#\1](https://github.com/cardano-foundation/cardano-wallet/pull/\1),g' \
>> artifacts/changes.md
done
38 changes: 38 additions & 0 deletions scripts/buildkite/release/push-artifacts.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
#! /usr/bin/env bash

set -euox pipefail

base_build=$(buildkite-agent meta-data get base-build)
NEW_GIT_TAG=$(buildkite-agent meta-data get release-version)

if [ "$RELEASE" == "false" ]; then
TAG=nightly
else
TAG=$NEW_GIT_TAG
fi

main_build=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-X GET "https://api.buildkite.com/v2/builds" \
| jq ".[] | select(.meta_data.\"triggered-by\" == \"$base_build\")" \
| jq .number)

mkdir -p artifacts

artifact() {
local artifact_name=$1
# shellcheck disable=SC2155
local artifact_value=$(curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
-X GET "https://api.buildkite.com/v2/organizations/cardano-foundation/pipelines/cardano-wallet/builds/$main_build/artifacts" \
| jq -r ".[] | select(.filename == \"$artifact_name\") \
| .download_url")
curl -H "Authorization: Bearer $BUILDKITE_API_TOKEN" -L \
-o "artifacts/$artifact_name" \
"$artifact_value"
gh release upload "$TAG" "artifacts/$artifact_name"
}

artifact "cardano-wallet-$NEW_GIT_TAG-linux64.tar.gz"
artifact "cardano-wallet.exe-$NEW_GIT_TAG-win64.zip"
artifact "cardano-wallet-$NEW_GIT_TAG-macos-silicon.tar.gz"
artifact "cardano-wallet-$NEW_GIT_TAG-macos-intel.tar.gz"
artifact "cardano-wallet-$NEW_GIT_TAG-docker-image.tgz"
41 changes: 41 additions & 0 deletions scripts/buildkite/release/push-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
#! /usr/bin/env bash

set -euox pipefail

NEW_GIT_TAG=$(buildkite-agent meta-data get release-version)

if [ "$RELEASE" == "false" ]; then
TAG=nightly
title="Nightly $NEW_GIT_TAG"
else
TAG=$NEW_GIT_TAG
title="Release $TAG"
fi

buildkite-agent artifact download artifacts/changes.md .
# shellcheck disable=SC2155
export CHANGES=$(cat artifacts/changes.md)

# shellcheck disable=SC2155
export NODE_TAG=$(buildkite-agent meta-data get node-tag)

# shellcheck disable=SC2034
export BUMP_CHANGES=xxx

# shellcheck disable=SC2034
export DOCKER_SHA=xxx

# shellcheck disable=SC2016
envsubst \
< "$RELEASE_SCRIPTS_DIR/release-template.md" \
> "$RELEASE_SCRIPTS_DIR/release-template-final.md"

if [ "$RELEASE" == "false" ]; then
gh release delete nightly --yes || true
fi

gh release create \
-d \
-F "$RELEASE_SCRIPTS_DIR/release-template-final.md" \
-t "$title" \
"$TAG"
26 changes: 26 additions & 0 deletions scripts/buildkite/release/push-tag.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#! /usr/bin/env bash

set -euox pipefail

RELEASE_GIT_COMMIT=$(buildkite-agent meta-data get release-candidate-commit)

git tag -l | xargs git tag -d
git fetch --tags

if [ "$RELEASE" == "false" ]; then
git tag -d nightly || true
TAG=nightly
else
TAG=$(buildkite-agent meta-data get release-version)
exists=$(git tag -l "$TAG")
if [ -n "$exists" ]; then
echo "Tag $TAG already exists. Remove it before proceeding."
exit 1
fi
fi

git tag -m "$TAG" "$TAG" "$RELEASE_GIT_COMMIT"

git remote set-url origin "https://$GITHUB_TOKEN@github.com/cardano-foundation/cardano-wallet.git"

git push origin -f "$TAG"
2 changes: 1 addition & 1 deletion scripts/buildkite/release/push-to-bump.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#! /bin/bash
#! /usr/bin/env bash

set -euox pipefail

Expand Down
8 changes: 7 additions & 1 deletion scripts/buildkite/release/release-candidate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,8 @@ tag_cabal_ver() {
tag_date "$1" | sed -e s/-0/-/g -e s/-/./g
}

git fetch --tags --force
git tag -l | xargs git tag -d
git fetch --tags

BASE_COMMIT=$(git rev-parse HEAD)

Expand All @@ -28,6 +29,8 @@ NEW_CABAL_VERSION=$(tag_cabal_ver "$NEW_GIT_TAG")

OLD_GIT_TAG=$( git tag -l "v2*-*-*" | sort | tail -n1)

LAST_RELEASE_DATE=$(tag_date "$OLD_GIT_TAG")

OLD_CABAL_VERSION=$(tag_cabal_ver "$OLD_GIT_TAG")

CARDANO_NODE_TAG=$(cardano-node version | head -n1 | awk '{print $2}')
Expand Down Expand Up @@ -72,3 +75,6 @@ buildkite-agent meta-data set "release-candidate-commit" "$RELEASE_COMMIT"
buildkite-agent meta-data set "release-candidate-branch" "$RELEASE_CANDIDATE_BRANCH"
buildkite-agent meta-data set "release-cabal-version" "$NEW_CABAL_VERSION"
buildkite-agent meta-data set "test-rc" "$TEST_RC"
buildkite-agent meta-data set "base-build" "$BUILDKITE_BUILD_ID"
buildkite-agent meta-data set "node-tag" "$CARDANO_NODE_TAG"
buildkite-agent meta-data set "last-release-date" "$LAST_RELEASE_DATE"
Loading

0 comments on commit 5ba3cbc

Please sign in to comment.