diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000..9b5039dd --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,106 @@ +name: Nightly + +on: + workflow_dispatch: + inputs: + dist_root: + description: 'DIST_ROOT' + required: true + default: '/ipns/dist.ipfs.io' + schedule: + - cron: '0 5 * * *' # UTC + +env: + DIST_ROOT: ${{ github.event.inputs.custom_dist_root || '/ipns/dist.ipfs.io' }} # content root used for calculating diff to build + GO_IPFS_VER: 'v0.9.1' # go-ipfs daemon used for chunking and applying diff + CLUSTER_CTL_VER: 'v0.14.0' # ipfs-cluster-ctl used for pinning + +concurrency: + group: nightly + cancel-in-progress: true + +jobs: + prepare-matrix: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - id: set-matrix + run: echo "::set-output name=matrix::$(jq -nc '$ARGS.positional' --args $(ls ./dists -1))" + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + + build: + runs-on: "ubuntu-latest" + needs: prepare-matrix + strategy: + fail-fast: false + matrix: + dist_name: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v2 + - name: Setup IPFS + run: ./scripts/ci/setup-ipfs.sh + env: + CLUSTER_USER: ${{ secrets.CLUSTER_USER }} + CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }} + timeout-minutes: 5 + - run: cd ./dists/${{ matrix.dist_name }} && make nightly + - run: ./dockerized make publish + - name: Create issue if build failed + uses: actions/github-script@v4 + if: ${{ failure() }} + with: + script: | + const title = 'Nightly build failed for ${{ matrix.dist_name }}' + const body = '${{ matrix.dist_name }} failed to build from the latest commit: https://github.com/ipfs/distributions/actions/runs/${{ github.run_id }}' + const opts = { owner: context.repo.owner, repo: context.repo.repo } + const response = await github.search.issuesAndPullRequests({ + q: `repo:ipfs/distributions is:issue is:open in:title ${title}` + }) + console.log('github.issuesAndPullRequests', response) + let link + if (response.data.items.length === 0) { + const created = await github.issues.create({ ...opts, title, body, + labels: ['kind/bug', 'need/triage'] + }) + console.log('no open issues, created a new one', created) + link = created.data.html_url + } + for (const issue of response.data.items) { + if (issue.title !== title) continue + console.log('found existing open issue', issue) + const created = await github.issues.createComment({ ...opts, + issue_number: issue.number, + body + }) + console.log('commented on existing open issue', created) + link = created.data.html_url + } + await github.repos.createCommitStatus({ ...opts, + sha: '${{ github.sha }}', + state: 'error', + target_url: link, + context: 'Problem with ${{ matrix.dist_name }}', + description: 'See details in the linked issue' + }) + - name: Inspect git status and contents of ./releases + run: git status && ls -Rhl ./releases + - name: Read CID of updated DAG + id: cid-reader + run: echo "::set-output name=CID::$(tail -1 ./versions)" + - name: Pin new website to ipfs-websites.collab.ipfscluster.io + run: ./scripts/ci/pin-to-cluster.sh + env: + PIN_CID: ${{ steps.cid-reader.outputs.CID }} + PIN_NAME: "ipfs/distributions/nightly/${{ matrix.dist_name }}" + PIN_ADD_EXTRA_ARGS: "--expire-in 168h" + CLUSTER_USER: ${{ secrets.CLUSTER_USER }} + CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }} + timeout-minutes: 60 + - name: Update PR status with preview link + run: ./scripts/ci/github-preview-link.sh + env: + GITHUB_TITLE: "Preview for ${{ matrix.dist_name }}" + CONTENT_PATH: "/ipfs/${{ steps.cid-reader.outputs.CID }}/${{ matrix.dist_name }}" + GIT_REVISION: ${{ github.sha }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Makefile b/Makefile index e18e6d30..97b6d599 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,31 @@ all: deps releases all_dists site -.PHONY: all all_dists deps -all_dists: $(notdir $(wildcard dists/*)) +DISTS = $(notdir $(wildcard dists/*)) -%: +NDISTS = $(DISTS:%=nightly-%) + +.PHONY: all all_dists deps nightly +all_dists: $(DISTS) + +$(DISTS): @echo "** $@ **" $(MAKE) -C dists/$@ @echo "" +nightly: $(NDISTS) + +$(NDISTS): + @echo "** $@ Nightly **" + $(MAKE) -C dists/$(@:nightly-%=%) nightly + @echo "" + deps: ./deps-check.sh releases: mkdir -p releases + .PHONY: site site: deps @echo "** Building site **" diff --git a/build-go.sh b/build-go.sh index 15ccd1ae..2c5e1412 100755 --- a/build-go.sh +++ b/build-go.sh @@ -199,7 +199,7 @@ function printInitialDistfile() { "name": "$distname", "owner": "$(< repo-owner)", "description": "$(< description)", - "date": "$(date '+%B %d, %Y')", + "date": "$(date -u '+%B %d, %Y')", "platforms": {} } EOF @@ -211,7 +211,7 @@ function printBuildInfo() { go version echo "git sha of code: $commit" uname -a - echo "built on $(date)" + echo "built on $(date -u)" } function buildWithMatrix() { @@ -219,7 +219,7 @@ function buildWithMatrix() { local package=$2 local output=$3 local commit=$4 - local version=$5 + local buildVersion=$5 test -n "$output" || fail "error: output dir not specified" test -e "$matfile" || fail "build matrix $matfile does not exist" @@ -229,13 +229,13 @@ function buildWithMatrix() { local distname distname=$(basename "$(pwd)") - printInitialDistfile "$distname" "$version" > dist.json + printInitialDistfile "$distname" "$buildVersion" > dist.json printBuildInfo "$commit" > "$output/build-info" # build each os/arch combo while read -r goos goarch do - doBuild "$goos" "$goarch" "$package" "$output" "$version" + doBuild "$goos" "$goarch" "$package" "$output" "$buildVersion" done < "$matfile" # build the source @@ -251,19 +251,42 @@ function cleanRepo() { git -C "$repopath" reset --hard } +function nightlyRevision() { + local repopath=$1 + local version=$2 + + # Use default branch, may be master, main or some other name + default_branch="$(git -C "$repopath" symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')" + # Find the last commit before nightly cut-off + cutoff_date="${version#nightly-}" + git -C "$repopath" rev-list -1 --first-parent --before="$cutoff_date" "$default_branch" +} + function checkoutVersion() { local repopath=$1 - local ref=$2 + local version=$2 test -n "$repopath" || fail "checkoutVersion: no repo to check out specified" - echo "==> checking out version $ref in $repopath" - cleanRepo "$repopath" + case $version in + nightly*) + ref="$(nightlyRevision "$repopath" "$version")" + ;; + *) + ref=$version + + # If there is a vtag, then checkout using / + # ('vtag' file is used for indicating 'submodule' + # such as 'fs-repo-migrations/fs-repo-0-to-1', + # those have release tags like 'fs-repo-0-to-1/v1.0.0') + if [ -e vtag ]; then + ref="$(cat vtag)/${version}" + fi + ;; + esac - # If there is a vtag, then checkout using / - if [ -e vtag ]; then - ref="$(cat vtag)/${ref}" - fi + echo "==> checking out version $version (git: $ref) in $repopath" + cleanRepo "$repopath" git -C "$repopath" checkout "$ref" > /dev/null || fail "failed to check out $ref in $reporoot" } @@ -338,7 +361,11 @@ function startGoBuilds() { fi echo "comparing $versions with $existing/$distname/versions" - newVersions=$(comm --nocheck-order -13 <(ipfs cat "$existing/$distname/versions") "$versions") + + existingVersions=$(mktemp) + ipfs cat "$existing/$distname/versions" > "$existingVersions" + + newVersions=$(comm --nocheck-order -13 "$existingVersions" "$versions") if [ -z "$newVersions" ]; then notice "skipping $distname - all versions published at $existing" @@ -369,11 +396,22 @@ function startGoBuilds() { while read -r version do - if [ -e "$outputDir/$version" ]; then + outputVersion=$outputDir/$version + + if [ -e "$outputVersion" ]; then echo "$version already exists, skipping..." continue fi + case $version in + nightly*) + buildVersion="$version-$(nightlyRevision "$repopath" "$version" | head -c 7)" + ;; + *) + buildVersion=$version + ;; + esac + notice "building version $version binaries" checkoutVersion "$repopath" "$version" installDeps "$repopath" > "deps-$version.log" 2>&1 @@ -394,13 +432,21 @@ function startGoBuilds() { go mod edit -require "$repo@$(git -C "$repopath" rev-parse HEAD)" fi - buildWithMatrix "$matfile" "$repo/$package" "$outputDir/$version" "$(currentSha "$repopath")" "$version" + buildWithMatrix "$matfile" "$repo/$package" "$outputVersion" "$(currentSha "$repopath")" "$buildVersion" echo "" done <<< "$newVersions" - cp "$versions" "$outputDir/versions" + # Additional cleanup/normalization for nightly builds + if grep -h ^nightly "$versions" "$existingVersions" > /dev/null; then + # Keep all tagged versions from repo + grep -v ^nightly "$versions" > "$outputDir/versions" + # Keep at most 7 nightly versions + grep -h ^nightly "$versions" "$existingVersions" | sort -ur | head -n7 >> "$outputDir/versions" + fi notice "build complete!" } startGoBuilds "$1" "$2" "$3" "$4" "$5" + +# vim: noet diff --git a/common.mk b/common.mk index 7bb757a5..86b5e78a 100644 --- a/common.mk +++ b/common.mk @@ -7,11 +7,15 @@ relpath = $(dir $(lastword $(MAKEFILE_LIST))) # Default values distname ?= $(notdir ${CURDIR}) releases ?= $(relpath)releases/${distname} +versions ?= versions + +nightlyVer = nightly-$(shell date -u '+%Y-%m-%d') + all: dist dist: - ${relpath}build-go.sh "${distname}" "${repo}" "${package}" versions + ${relpath}build-go.sh "${distname}" "${repo}" "${package}" "${versions}" update_sources: cd gopath/src/${repo} @@ -19,3 +23,7 @@ update_sources: clean: rm -rf $(releases) + +nightly: + grep -qxF ${nightlyVer} versions || echo ${nightlyVer} >> versions + diff --git a/dist.sh b/dist.sh index a1efafd8..7863396f 100755 --- a/dist.sh +++ b/dist.sh @@ -74,6 +74,7 @@ case $1 in case "$nvers" in *-*) echo "WARNING: not marking pre-release $dist $nvers as the current version." ;; + nightly) nvers=$nvers-$(date -u '+%Y-%m-%d') ;; *) echo "$nvers" > "dists/$dist/current" ;; esac diff --git a/scripts/ci/github-preview-link.sh b/scripts/ci/github-preview-link.sh index 41d92654..18d2c348 100755 --- a/scripts/ci/github-preview-link.sh +++ b/scripts/ci/github-preview-link.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash set -e -PREVIEW_URL="https://dweb.link$CONTENT_PATH" +PREVIEW_URL=${PREVIEW_URL:-"https://dweb.link$CONTENT_PATH"} API_PARAMS=$(jq --monochrome-output --null-input \ --arg state "success" \ --arg target_url "$PREVIEW_URL" \ - --arg description "Preview updated website on IPFS" \ - --arg context "Preview is ready" \ + --arg description "${GITHUB_DESCRIPTION:-"Preview updated website on IPFS"}" \ + --arg context "${GITHUB_TITLE:-"Preview is ready"}" \ '{ state: $state, target_url: $target_url, description: $description, context: $context }' ) curl --output /dev/null --silent --show-error \ -X POST -H "Authorization: Bearer $GITHUB_TOKEN" -H 'Content-Type: application/json' \