From cd78bdfdf8678995f8a3c4a21a53bc4aaed5025f Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 12 Jul 2021 16:37:48 +0200
Subject: [PATCH 01/34] wip: signing poc with hardcoded dist details

---
 .github/workflows/main.yml     | 218 ++++++++++++++++++++++++++++++---
 README.md                      |  20 +--
 build-go.sh                    |   7 +-
 dists/ipfs-update/build_matrix |  12 --
 dockerized                     |   2 +-
 scripts/dist.js                |   2 +-
 6 files changed, 217 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 0d528ea3..17243d31 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,40 +2,220 @@ name: CI
 
 on:
   push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
+# TODO: restore after testing
+#    branches:
+#      - master
+#  pull_request:
+#    branches:
+#      - master
+
+env:
+ # TODO: remove after testing
+  DIST_ROOT: '/ipfs/QmX6J3hVtMF9Y73CKcyLHfgviEfXGFFQABWPXsD17EsBhg' # dist.ipfs.io without ipfs-update@v1.7.1
 
 jobs:
   build:
     runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: '14'
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-      - run: sudo snap install ipfs jq
-      - run: ipfs init --profile server
-      - run: ipfs daemon &
+      - name: Install ipfs and deps
+        run: |
+          sudo snap install ipfs jq
+          ipfs init --profile server
       - name: Wait for ipfs daemon
-        run: npx wait-port http://127.0.0.1:8080/api/v0/version
+        run: ipfs daemon & npx wait-port http://127.0.0.1:8080/api/v0/version
       - name: Connect to ipfs cluster
         run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
-      - run: make publish
-      # todo: add $(cat versions) to cluster (and wait)
-      # todo: update dist dnslink if changed.
-  
-  lint: 
+      - name: Build any new ./releases
+        run: ./dockerized make all_dists
+      - name: Inspect git status and contents of ./releases
+        run: git status && ls -Rhl ./releases
+      - name: Temporarily save ./releases artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: releases-unsigned-diff
+          path: releases
+          retention-days: 1
+
+  lint:
     runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
+      - uses: actions/setup-node@v2
         with:
           node-version: '14'
       - run: npm ci --no-audit --progress=false
       - run: npm run lint
+
+  sign-macos:
+    runs-on: "macos-latest"
+    needs: build
+    steps:
+      - uses: actions/checkout@v2
+      - name: Retrieve unsigned artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: releases-unsigned-diff
+          path: releases
+      - name: List ./releases before
+        run: ls -Rhl ./releases
+      - name: Install gon via HomeBrew for code signing and app notarization
+        run: |
+          brew tap mitchellh/gon
+          brew install ipfs coreutils gawk gnu-sed jq mitchellh/gon/gon
+          ipfs init --profile server # needed for calculating NEW_CID later
+      - name: Import Keychain Certs
+        uses: apple-actions/import-codesign-certs@253ddeeac23f2bdad1646faac5c8c2832e800071 # v1@2020-02-03
+        with:
+          p12-file-base64: ${{ secrets.APPLE_CERTS_P12 }}
+          p12-password: ${{ secrets.APPLE_CERTS_PASS }}
+      - name: Verify identity used for signing
+        run: security find-identity -v
+      - name: Unpack any new darwin arm64 and amd64 binaries to ./tmp
+        run: |
+          # ./releases/{DIST_NAME}/{DIST_VERSION}/*_darwin-${arch}.tar.gz -> ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
+          for NEW_DIR in ./releases/*/v*; do
+            (! test -d "$NEW_DIR") && continue
+            DIST_VERSION=$(basename "$NEW_DIR")
+            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+            for arch in "amd64" "arm64"; do
+              mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned"
+              tar -zxvf "./releases/${DIST_NAME}/${DIST_VERSION}/${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz" -C "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/"
+            done
+          done
+          ls -Rhl ./tmp
+      - name: Sign and notarize the mac binaries
+        env:
+          AC_USERNAME: ${{ secrets.APPLE_AC_USERNAME }} # implicitly read from env by gon
+          AC_PASSWORD: ${{ secrets.APPLE_AC_PASSWORD }}
+        run: |
+          # Find and sign executables in ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
+          for NEW_DIR in ./releases/*/v*; do
+            (! test -d "$NEW_DIR") && continue
+            DIST_VERSION=$(basename "$NEW_DIR")
+            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+            for arch in "amd64" "arm64"; do
+              EXECUTABLES=$(jq -nc '$ARGS.positional' --args $(find "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/" -perm +111 -type f -print))
+              echo "{
+                  \"source\" : $EXECUTABLES,
+                  \"bundle_id\" : \"io.ipfs.dist.${DIST_NAME}\",
+                  \"apple_id\": {
+                    \"password\":  \"@env:AC_PASSWORD\"
+                  },
+                  \"sign\" :{
+                    \"application_identity\" : \"Developer ID Application: Protocol Labs, Inc. (7Y229E2YRL)\"
+                  },
+                  \"zip\" :{
+                      \"output_path\" : \"./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip\"
+                  }
+              }" | tee | jq > "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
+              gon -log-level=info -log-json "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
+            done
+          done
+      - name: Temporarily save ./tmp
+        uses: actions/upload-artifact@v2
+        with:
+          name: tmp
+          path: ./tmp/
+          retention-days: 1
+      - name: Update changed binaries in ./releases
+        run: |
+          for NEW_DIR in ./releases/*/v*; do
+            (! test -d "$NEW_DIR") && continue
+            DIST_VERSION=$(basename "$NEW_DIR")
+            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+            for arch in "amd64" "arm64"; do
+              echo "-> Starting the update of darwin_${arch}.tar.gz for name='${DIST_NAME}' and version='${DIST_VERSION}'"
+              # unzip signed binaries to a directory matching .tar.gz structure
+              cd "${{ github.workspace }}"
+              mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}"
+              cd "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}/"
+              echo "-> Unpacking gon .zip for ${arch}"
+              unzip "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip"
+              echo "-> Unpacked contents"
+              ls -Rhl "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/"
+              # replace .tar.gz with one that has the same structure, but signed binaries
+              PKG_NAME="${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz"
+              PKG_ROOT="${{ github.workspace }}/releases/${DIST_NAME}/${DIST_VERSION}"
+              PKG_PATH="${PKG_ROOT}/${PKG_NAME}"
+              DIST_JSON="${PKG_ROOT}/dist.json"
+              # read old hashes
+              OLD_CID=$(cat "${PKG_PATH}.cid")
+              OLD_SHA512=$(gawk '{ print $1; }' < "${PKG_PATH}.sha512")
+              echo "-> Found old $PKG_NAME"
+              echo "   old CID:    $OLD_CID"
+              echo "   old SHA512: $OLD_SHA512"
+              echo "-> Updating $PKG_NAME"
+              rm "$PKG_PATH"
+              tar -czvf "${{ github.workspace }}/releases/${DIST_NAME}/${DIST_VERSION}/$PKG_NAME" -C "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/" "${DIST_NAME}"
+              # calculate new hashes
+              NEW_CID=$(ipfs add -Qn "$PKG_PATH")
+              NEW_SHA512_LINE=$(gsha512sum "$PKG_PATH")
+              NEW_SHA512=$(echo "$NEW_SHA512_LINE" | gawk '{ print $1; }')
+              echo "-> New $PKG_NAME"
+              echo "   new CID:    $NEW_CID"
+              echo "   new SHA512: $NEW_SHA512"
+              # update metadata to use new hashes
+              echo "$NEW_CID" > "${PKG_PATH}.cid"
+              echo "$NEW_SHA512_LINE" > "${PKG_PATH}.sha512"
+              gsed -i "s/${OLD_CID}/${NEW_CID}/g; s/${OLD_SHA512}/${NEW_SHA512}/g" "${PKG_ROOT}/dist.json"
+              echo "-> Completed the update of ${arch}.tar.gz for ${DIST_NAME} ${DIST_VERSION}"
+            done
+          done
+      - name: List ./releases after
+        run: ls -Rhl ./releases
+      - name: Temporarily save notarized artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: releases-signed-macos-diff
+          path: releases
+          retention-days: 1
+
+  persist:
+    runs-on: "ubuntu-latest"
+    needs: sign-macos
+    steps:
+      - uses: actions/checkout@v2
+      - name: Retrieve signed artifacts
+        uses: actions/download-artifact@v2
+        with:
+          name: releases-signed-macos-diff
+          path: releases
+      - name: List ./releases
+        run: ls -Rhl ./releases
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+      - name: Install ipfs and deps
+        run: |
+          sudo snap install ipfs jq
+          ipfs init --profile server
+          # DNS over HTTPS to avoid random Github Action failures like
+          # Error: lookup _dnsaddr.ipfs-websites.collab.ipfscluster.io on 127.0.0.53:53: dial udp 127.0.0.53:53: i/o timeout
+          ipfs config --json DNS.Resolvers '{ ".": "https://cloudflare-dns.com/dns-query" }'
+      - name: Wait for ipfs daemon
+        run: ipfs daemon & npx wait-port http://127.0.0.1:8080/api/v0/version
+      - name: Connect to cluster.ipfs.io
+        run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
+      - run: make publish
+      - run: git status
+      - name: Read CID of updated DAG
+        id: cid-reader
+        run: echo "::set-output name=CID::$(tail -1 ./versions)"
+      # pin new root to cluster + update PR status with preview link
+      - name: Connect to ipfs-websites.collab.ipfscluster.io
+        run: ipfs swarm connect /dnsaddr/ipfs-websites.collab.ipfscluster.io
+      - uses: ipfs-shipyard/ipfs-github-action@bda86e6563a66edcea22a0b0d20db6f19a2b5899 # https://github.com/ipfs-shipyard/ipfs-github-action/pull/7
+        id: ipfs
+        with:
+          path_to_add: /ipfs/${{ steps.cid-reader.outputs.CID }}
+          cluster_host: /dnsaddr/ipfs-websites.collab.ipfscluster.io
+          cluster_user: ${{ secrets.CLUSTER_USER }}
+          cluster_password: ${{ secrets.CLUSTER_PASSWORD }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - run: echo ${{ steps.ipfs.outputs.url }}
+      - run: echo ${{ github.ref }}
diff --git a/README.md b/README.md
index 9363ea4d..a438f817 100644
--- a/README.md
+++ b/README.md
@@ -90,6 +90,9 @@ Example:
 > ./dist.sh add-version fs-repo-99-to-100 v1.0.1
 ```
 
+**Official build:** If you want to build official signed binaries for added version, commit changes to `dists/<dist>` and open a PR against `ipfs/distributions`.
+Github Action workflow for a PR builds and signs new version using deterministic toolchain and spits out updated root CID at the end.
+
 ### Adding a new (go) distribution
 
 Run:
@@ -107,25 +110,26 @@ The optional `sub_package` argument is used to specify a module within a repo.
 
 ### Publishing
 
-In the root of the repository, run:
+To produce a CID (`<NEW_HASH>`) that includes binaries for all versions defined in `./dists/`, in the root of the repository, run:
 
 ```sh
 > make publish
 ```
 
-This will build any new binaries defined by dist and the website to the `releases` dir, add it to ipfs and patch it into the existing dag for the published dist.ipfs.io. Save the hash it spits out (we'll call it `<NEW_HASH>`), that's the new hash for `dists.ipfs.io`. We also append it to a file called `versions` in the repo root (*not* checked into git).
-
-Next, you should probably:
+- This will build any new binaries defined by dist and the website to the `releases` dir, add it to ipfs and patch it into the existing dag for the published `/ipns/dist.ipfs.io`.
+- Versions that are already present on the website will be reused, speeding up the build.
+- Updated CID (`<NEW_HASH>`) will be printed at the end. That's the new hash for `dists.ipfs.io`. We also append it to a file called `versions` in the repo root (*not* checked into git).
 
-1. Load the dists website in your browser to make sure everything looks right: `http://127.0.0.1:8080/ipfs/<NEW_HASH>`.
-2. Compare `<NEW_HASH>` with the current `dists.ipfs.io` to make sure nothing is amiss: `ipfs object diff /ipns/dist.ipfs.io /ipfs/<NEW_HASH>`
+After the local build is done, make a quick inspection:
 
-If all looks well, **pin the hash using pinbot** (#ipfs-pinbot on Freenode, ask someone if you don't have permission to do so).
+2. Load the dists website in your browser to make sure everything looks right: `http://127.0.0.1:8080/ipfs/<NEW_HASH>`.
+3. Compare `<NEW_HASH>` with the current `dists.ipfs.io` to make sure nothing is amiss: `ipfs object diff /ipns/dist.ipfs.io /ipfs/<NEW_HASH>`
 
 Finally,
 
 1. Commit your changes and make a PR. Specifically, the changes to `dists/<dist>/versions` and `dists/<dist>/current`.
-2. Make a PR with an edit on [protocol/infra](https://github.com/protocol/infra/blob/master/dns/config/dist.ipfs.io.yaml) with the hash you got from `make publish` and a link to the PR above.
+2. Wait for [Github Action](https://github.com/ipfs/distributions/actions/) on your PR to build and **signed** binaries. `<NEW_SIGNED_HASH>` will be different than one from local build.
+3. Make a PR with an edit on [protocol/infra](https://github.com/protocol/infra/blob/master/dns/config/dist.ipfs.io.yaml) with `<NEW_SIGNED_HASH>` you got from the Github Action output and a link to the PR above.
 
 If you have permission, you can just merge the PR, update the DNS, and then immediately, close the issue on ipfs/infrastructure. Ping someone on IRC.
 
diff --git a/build-go.sh b/build-go.sh
index caed0928..8e0a6cf2 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -9,8 +9,9 @@ export GOPATH
 # Always use go modules
 export GO111MODULE=on
 
-DIST_PATH=${DIST_PATH:-/ipns/dist.ipfs.io}
-DIST_PATH=$(ipfs resolve "$DIST_PATH")
+# Content path to use when looking for pre-existing release data
+DIST_ROOT=${DIST_ROOT:-/ipns/dist.ipfs.io}
+DIST_ROOT=$(ipfs resolve "$DIST_ROOT")
 
 # normalize umask
 umask 022
@@ -334,7 +335,7 @@ function startGoBuilds() {
 	fi
 
 	if [ -z "$existing" ]; then
-		existing="$DIST_PATH"
+		existing="$DIST_ROOT"
 	fi
 
 	echo "comparing $versions with $existing/$distname/versions"
diff --git a/dists/ipfs-update/build_matrix b/dists/ipfs-update/build_matrix
index c562e61b..a523aa54 100644
--- a/dists/ipfs-update/build_matrix
+++ b/dists/ipfs-update/build_matrix
@@ -1,14 +1,2 @@
 darwin amd64
 darwin arm64
-freebsd 386
-freebsd amd64
-freebsd arm
-openbsd 386
-openbsd amd64
-openbsd arm
-linux 386
-linux amd64
-linux arm
-linux arm64
-windows 386
-windows amd64
diff --git a/dockerized b/dockerized
index 80c296cc..d5f15c85 100755
--- a/dockerized
+++ b/dockerized
@@ -9,4 +9,4 @@ docker build . -t distributions --build-arg CACHEBUST=`date --iso-8601=date` --b
 
 # We use host networking as the build process assumes a fairly long-lived ipfs
 # node has the CIDs (we give them to the collab cluster to pin) 
-docker run --rm -it --network host -v `pwd`:/build distributions "$@"
+docker run --rm -i --network host -e DIST_ROOT -v `pwd`:/build distributions "$@"
diff --git a/scripts/dist.js b/scripts/dist.js
index 4dbfdd52..6f51851c 100755
--- a/scripts/dist.js
+++ b/scripts/dist.js
@@ -18,7 +18,7 @@ require('make-promises-safe') // exit on error
 const RELEASE_PATH = join(__dirname, '..', 'releases')
 const SITE_PATH = join(__dirname, '..', 'site', 'data', 'releases')
 const DIST_PATH = join(__dirname, '..', 'dists')
-const DIST_ROOT = '/ipns/dist.ipfs.io'
+const DIST_ROOT = process.env.DIST_ROOT || '/ipns/dist.ipfs.io'
 
 const ipfs = IpfsHttpClient()
 

From 230d1cd648ea63484e8da466fdb0f607a86cb9ff Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Wed, 21 Jul 2021 17:18:01 +0100
Subject: [PATCH 02/34] feat: create nightly builds

---
 .github/workflows/nightly.yml | 26 ++++++++++++++++++++++++++
 Makefile                      | 18 +++++++++++++++---
 build-go.sh                   | 34 ++++++++++++++++++++++++++++------
 common.mk                     | 10 +++++++++-
 dist.sh                       |  1 +
 5 files changed, 79 insertions(+), 10 deletions(-)
 create mode 100644 .github/workflows/nightly.yml

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
new file mode 100644
index 00000000..78a007b0
--- /dev/null
+++ b/.github/workflows/nightly.yml
@@ -0,0 +1,26 @@
+name: Nightly
+
+on:
+  schedule:
+    - cron: '30/15 12 * * *' # UTC
+
+jobs:
+  build:
+    runs-on: "ubuntu-latest"
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v2-beta
+        with:
+          node-version: '14'
+      - uses: actions/setup-go@v2
+        with:
+          go-version: '1.16'
+      - run: sudo snap install ipfs jq
+      - run: ipfs init --profile server
+      - run: ipfs daemon &
+      - name: Wait for ipfs daemon
+        run: npx wait-port http://127.0.0.1:8080/api/v0/version
+      - name: Connect to ipfs cluster
+        run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
+      - run: make deps nightly all_dists
+      #- run: make publish
diff --git a/Makefile b/Makefile
index e18e6d30..97b6d599 100644
--- a/Makefile
+++ b/Makefile
@@ -1,19 +1,31 @@
 all: deps releases all_dists site
 
-.PHONY: all all_dists deps
-all_dists: $(notdir $(wildcard dists/*))
+DISTS = $(notdir $(wildcard dists/*))
 
-%:
+NDISTS = $(DISTS:%=nightly-%)
+
+.PHONY: all all_dists deps nightly
+all_dists: $(DISTS)
+
+$(DISTS):
 	@echo "** $@ **"
 	$(MAKE) -C dists/$@
 	@echo ""
 
+nightly: $(NDISTS)
+
+$(NDISTS):
+	@echo "** $@ Nightly **"
+	$(MAKE) -C dists/$(@:nightly-%=%) nightly
+	@echo ""
+
 deps:
 	./deps-check.sh
 
 releases:
 	mkdir -p releases
 
+
 .PHONY: site
 site: deps
 	@echo "** Building site **"
diff --git a/build-go.sh b/build-go.sh
index caed0928..ccede55e 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -219,7 +219,7 @@ function buildWithMatrix() {
 	local package=$2
 	local output=$3
 	local commit=$4
-	local version=$5
+	local buildVersion=$5
 
 	test -n "$output" || fail "error: output dir not specified"
 	test -e "$matfile" || fail "build matrix $matfile does not exist"
@@ -229,13 +229,13 @@ function buildWithMatrix() {
 	local distname
 	distname=$(basename "$(pwd)")
 
-	printInitialDistfile "$distname" "$version" > dist.json
+	printInitialDistfile "$distname" "$buildVersion" > dist.json
 	printBuildInfo "$commit" > "$output/build-info"
 
 	# build each os/arch combo
 	while read -r goos goarch
 	do
-		doBuild "$goos" "$goarch" "$package" "$output" "$version"
+		doBuild "$goos" "$goarch" "$package" "$output" "$buildVersion"
 	done < "$matfile"
 
 	# build the source
@@ -253,10 +253,21 @@ function cleanRepo() {
 
 function checkoutVersion() {
 	local repopath=$1
-	local ref=$2
+	local version=$2
 
 	test -n "$repopath" || fail "checkoutVersion: no repo to check out specified"
 
+
+	case $version in
+	nightly*)
+		# Use default branch, may be master, main or some other name
+		ref=$(git -C "$repopath" symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
+		;;
+	*)
+		ref=$version
+		;;
+	esac
+
 	echo "==> checking out version $ref in $repopath"
 	cleanRepo "$repopath"
 
@@ -369,11 +380,22 @@ function startGoBuilds() {
 
 	while read -r version
 	do
-		if [ -e "$outputDir/$version" ]; then
+		outputVersion=$outputDir/$version
+
+		if [ -e "$outputVersion" ]; then
 			echo "$version already exists, skipping..."
 			continue
 		fi
 
+		case $version in
+		nightly*)
+			buildVersion="$version-$(git rev-parse --short=7 HEAD)"
+			;;
+		*)
+			buildVersion=$version
+			;;
+		esac
+
 		notice "building version $version binaries"
 		checkoutVersion "$repopath" "$version"
 		installDeps "$repopath" > "deps-$version.log" 2>&1
@@ -394,7 +416,7 @@ function startGoBuilds() {
 		    go mod edit -require "$repo@$(git -C "$repopath" rev-parse HEAD)"
 		fi
 
-		buildWithMatrix "$matfile" "$repo/$package" "$outputDir/$version" "$(currentSha "$repopath")" "$version"
+		buildWithMatrix "$matfile" "$repo/$package" "$outputVersion" "$(currentSha "$repopath")" "$buildVersion"
 		echo ""
 	done <<< "$newVersions"
 
diff --git a/common.mk b/common.mk
index 7bb757a5..200afa7d 100644
--- a/common.mk
+++ b/common.mk
@@ -7,11 +7,15 @@ relpath = $(dir $(lastword $(MAKEFILE_LIST)))
 # Default values
 distname ?= $(notdir ${CURDIR})
 releases ?= $(relpath)releases/${distname}
+versions ?= versions
+
+nightlyVer = nightly-$(shell date '+%Y-%m-%d')
+
 
 all: dist
 
 dist:
-	${relpath}build-go.sh "${distname}" "${repo}" "${package}" versions
+	${relpath}build-go.sh "${distname}" "${repo}" "${package}" "${versions}"
 
 update_sources:
 	cd gopath/src/${repo}
@@ -19,3 +23,7 @@ update_sources:
 
 clean:
 	rm -rf $(releases)
+
+nightly:
+	grep -qxF ${nightlyVer} versions || echo ${nightlyVer} >> versions
+
diff --git a/dist.sh b/dist.sh
index a1efafd8..d07485d4 100755
--- a/dist.sh
+++ b/dist.sh
@@ -74,6 +74,7 @@ case $1 in
 
 		case "$nvers" in
 			*-*) echo "WARNING: not marking pre-release $dist $nvers as the current version." ;;
+			nightly) nvers=$nvers-$(date '+%Y-%m-%d') ;;
 			*) echo "$nvers" > "dists/$dist/current" ;;
 		esac
 

From 1bb2f200839f7495c9952c81427b3e36e3f98bb9 Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Thu, 22 Jul 2021 13:30:17 +0100
Subject: [PATCH 03/34] Ensure nightly dates use utc

---
 common.mk | 2 +-
 dist.sh   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/common.mk b/common.mk
index 200afa7d..86b5e78a 100644
--- a/common.mk
+++ b/common.mk
@@ -9,7 +9,7 @@ distname ?= $(notdir ${CURDIR})
 releases ?= $(relpath)releases/${distname}
 versions ?= versions
 
-nightlyVer = nightly-$(shell date '+%Y-%m-%d')
+nightlyVer = nightly-$(shell date -u '+%Y-%m-%d')
 
 
 all: dist
diff --git a/dist.sh b/dist.sh
index d07485d4..7863396f 100755
--- a/dist.sh
+++ b/dist.sh
@@ -74,7 +74,7 @@ case $1 in
 
 		case "$nvers" in
 			*-*) echo "WARNING: not marking pre-release $dist $nvers as the current version." ;;
-			nightly) nvers=$nvers-$(date '+%Y-%m-%d') ;;
+			nightly) nvers=$nvers-$(date -u '+%Y-%m-%d') ;;
 			*) echo "$nvers" > "dists/$dist/current" ;;
 		esac
 

From 7208eae60c8a97d4d783e25477a72f04c0bc1999 Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Thu, 22 Jul 2021 16:18:13 +0100
Subject: [PATCH 04/34] Allow manual invocation of nightly workflow

---
 .github/workflows/nightly.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 78a007b0..a24c0778 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -1,8 +1,10 @@
 name: Nightly
 
 on:
+  ## Allow manual invocation
+  workflow_dispatch:
   schedule:
-    - cron: '30/15 12 * * *' # UTC
+    - cron: '0 5 * * *' # UTC
 
 jobs:
   build:

From 42531b669e958cb16c37ff92f2a75edaaf87e3f6 Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Fri, 23 Jul 2021 12:16:38 +0100
Subject: [PATCH 05/34] Keep last 7 nightly versions in published versions file

---
 build-go.sh | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index ccede55e..30202d24 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -349,7 +349,11 @@ function startGoBuilds() {
 	fi
 
 	echo "comparing $versions with $existing/$distname/versions"
-	newVersions=$(comm --nocheck-order -13 <(ipfs cat "$existing/$distname/versions") "$versions")
+
+	outputDir="$releases/$distname"
+
+	ipfs cat "$existing/$distname/versions" > "$outputDir/existingVersions"
+	newVersions=$(comm --nocheck-order -13 "$outputDir/existingVersions" "$versions")
 
 	if [ -z "$newVersions" ]; then
 		notice "skipping $distname - all versions published at $existing"
@@ -357,7 +361,6 @@ function startGoBuilds() {
 	fi
 	printVersions "$newVersions"
 
-	outputDir="$releases/$distname"
 
 	# if the output directory already exists, warn user
 	if [ -e "$outputDir" ]; then
@@ -420,7 +423,11 @@ function startGoBuilds() {
 		echo ""
 	done <<< "$newVersions"
 
-	cp "$versions" "$outputDir/versions"
+	# All tagged versions from repo
+	grep -v ^nightly "$versions" > "$outputDir/versions"
+
+	# Last 7 nightly versions from repo and existing
+	grep -h ^nightly "$versions" "$outputDir/existingVersions" | sort -ur | head -n7 >> "$outputDir/versions"
 
 	notice "build complete!"
 }

From 18ca56ee5710f40c1ce0f62b973236cee0223fdb Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Fri, 23 Jul 2021 12:22:46 +0100
Subject: [PATCH 06/34] Touch existing versions file to create it

---
 build-go.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/build-go.sh b/build-go.sh
index 30202d24..6186b574 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -352,7 +352,8 @@ function startGoBuilds() {
 
 	outputDir="$releases/$distname"
 
-	ipfs cat "$existing/$distname/versions" > "$outputDir/existingVersions"
+	touch "$outputDir/existingVersions"
+	ipfs cat "$existing/$distname/versions" >> "$outputDir/existingVersions"
 	newVersions=$(comm --nocheck-order -13 "$outputDir/existingVersions" "$versions")
 
 	if [ -z "$newVersions" ]; then

From 48704544e0e8056d7c7e58d0b112b36702c5de45 Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Fri, 23 Jul 2021 12:29:43 +0100
Subject: [PATCH 07/34] Create output dir earlier

---
 build-go.sh | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index 6186b574..9b90eb5a 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -351,7 +351,14 @@ function startGoBuilds() {
 	echo "comparing $versions with $existing/$distname/versions"
 
 	outputDir="$releases/$distname"
+	# if the output directory already exists, warn user
+	if [ -e "$outputDir" ]; then
+		warn "dirty output directory"
+		warn "will skip building already existing binaries"
+		warn "to perform a fresh build, please delete $outputDir"
+	fi
 
+	mkdir -p "$outputDir"
 	touch "$outputDir/existingVersions"
 	ipfs cat "$existing/$distname/versions" >> "$outputDir/existingVersions"
 	newVersions=$(comm --nocheck-order -13 "$outputDir/existingVersions" "$versions")
@@ -363,12 +370,6 @@ function startGoBuilds() {
 	printVersions "$newVersions"
 
 
-	# if the output directory already exists, warn user
-	if [ -e "$outputDir" ]; then
-		warn "dirty output directory"
-		warn "will skip building already existing binaries"
-		warn "to perform a fresh build, please delete $outputDir"
-	fi
 
 	export GOPATH
 	GOPATH="$(pwd)/gopath"

From 8e11abeff4c05e6acc74f37682eb66028cc9412a Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Thu, 22 Jul 2021 14:05:31 +0200
Subject: [PATCH 08/34] wip: debugging pinning to cluster

---
 .github/workflows/main.yml | 118 ++++++++++++++++++++++++++++++-------
 1 file changed, 98 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 17243d31..f8f26d1d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -23,12 +23,61 @@ jobs:
           node-version: '14'
       - name: Install ipfs and deps
         run: |
-          sudo snap install ipfs jq
-          ipfs init --profile server
+          sudo snap install jq
+          # set up go-ipfs
+          IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
+          tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
+          echo "${{ github.workspace }}/go-ipfs/" >> $GITHUB_PATH
+          # set up ipfs-cluster-ctl
+          CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
+          tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
+          echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+      - name: Set up ipfs
+        run: |
+          ipfs init --profile server,test,lowpower
+          # restore deterministic port (changed by test profile)
+          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
+          # DNS over HTTPS to avoid random Github Action failures like
+          # Error: lookup _dnsaddr.ipfs-websites.collab.ipfscluster.io on 127.0.0.53:53: dial udp 127.0.0.53:53: i/o timeout
+          ipfs config --json DNS.Resolvers '{ ".": "https://cloudflare-dns.com/dns-query" }'
       - name: Wait for ipfs daemon
-        run: ipfs daemon & npx wait-port http://127.0.0.1:8080/api/v0/version
+        run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+        timeout-minutes: 3
       - name: Connect to ipfs cluster
         run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
+      - name: List peers
+        run: |
+          echo 'resolv'
+          ls -l /etc/resolv.conf
+          cat /etc/resolv.conf
+          echo 'updated resolv'
+          # TODO: simplify
+          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
+          cat /etc/resolv.conf
+          echo 'list swarm peers'
+          ipfs swarm peers
+          echo 'list cluster peers at ipfs-websites'
+          ipfs-cluster-ctl \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            peers ls > cluster-peers-ls
+          for peerid in $(grep "IPFS:" cluster-peers-ls | awk '{print $3; }'); do
+            for maddr in $(grep "/p2p/${peerid}" cluster-peers-ls | awk '{ print $2; }'); do
+              ipfs swarm connect "$maddr" || true
+            done
+          done
+          echo 'list swarm peers with cluster ones'
+          ipfs swarm peers
+      - name: Test pin
+        run: |
+          echo 'pin test cid'
+          ipfs-cluster-ctl \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            --debug \
+            pin add QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR \
+            --wait --name  'test_${{ github.run_id }}' --expire-in 24h
+        timeout-minutes: 3
       - name: Build any new ./releases
         run: ./dockerized make all_dists
       - name: Inspect git status and contents of ./releases
@@ -191,13 +240,26 @@ jobs:
           node-version: '14'
       - name: Install ipfs and deps
         run: |
-          sudo snap install ipfs jq
-          ipfs init --profile server
+          sudo snap install jq
+          # set up go-ipfs
+          IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
+          tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
+          echo "${{ github.workspace }}/go-ipfs/" >> $GITHUB_PATH
+          # set up ipfs-cluster-ctl
+          CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
+          tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
+          echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+      - name: Set up ipfs
+        run: |
+          ipfs init --profile server,test,lowpower
+          # restore deterministic port (changed by test profile)
+          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
           # DNS over HTTPS to avoid random Github Action failures like
           # Error: lookup _dnsaddr.ipfs-websites.collab.ipfscluster.io on 127.0.0.53:53: dial udp 127.0.0.53:53: i/o timeout
           ipfs config --json DNS.Resolvers '{ ".": "https://cloudflare-dns.com/dns-query" }'
       - name: Wait for ipfs daemon
-        run: ipfs daemon & npx wait-port http://127.0.0.1:8080/api/v0/version
+        run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+        timeout-minutes: 3
       - name: Connect to cluster.ipfs.io
         run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
       - run: make publish
@@ -205,17 +267,33 @@ jobs:
       - name: Read CID of updated DAG
         id: cid-reader
         run: echo "::set-output name=CID::$(tail -1 ./versions)"
-      # pin new root to cluster + update PR status with preview link
-      - name: Connect to ipfs-websites.collab.ipfscluster.io
-        run: ipfs swarm connect /dnsaddr/ipfs-websites.collab.ipfscluster.io
-      - uses: ipfs-shipyard/ipfs-github-action@bda86e6563a66edcea22a0b0d20db6f19a2b5899 # https://github.com/ipfs-shipyard/ipfs-github-action/pull/7
-        id: ipfs
-        with:
-          path_to_add: /ipfs/${{ steps.cid-reader.outputs.CID }}
-          cluster_host: /dnsaddr/ipfs-websites.collab.ipfscluster.io
-          cluster_user: ${{ secrets.CLUSTER_USER }}
-          cluster_password: ${{ secrets.CLUSTER_PASSWORD }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - run: echo ${{ steps.ipfs.outputs.url }}
-      - run: echo ${{ github.ref }}
+      - name: Pin new website to ipfs-websites.collab.ipfscluster.io
+        run: |
+          ipfs swarm connect /dnsaddr/ipfs-websites.collab.ipfscluster.io
+          echo 'list swarm peers'
+          ipfs swarm peers
+          echo 'list cluster peers'
+          ipfs-cluster-ctl \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            peers ls
+          echo 'pin to cluster'
+          ipfs-cluster-ctl \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            pin add \
+            --wait --debug \
+            --pin-name="https://github.com/ipfs/distributions/commits/${{ github.sha }}" \
+            "${{ steps.cid-reader.outputs.CID }}"
+      # TODO: update PR status with preview link
+      # - uses: ipfs-shipyard/ipfs-github-action@51639f910548de983b3f49b209bc23e0cf997b14 # https://github.com/ipfs-shipyard/ipfs-github-action/pull/7
+      #  id: ipfs
+      #  with:
+      #    path_to_add: /ipfs/${{ steps.cid-reader.outputs.CID }}
+      #    cluster_host: /dnsaddr/ipfs-websites.collab.ipfscluster.io
+      #    cluster_user: ${{ secrets.CLUSTER_USER }}
+      #    cluster_password: ${{ secrets.CLUSTER_PASSWORD }}
+      #  env:
+      #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      #- run: echo ${{ steps.ipfs.outputs.url }}
+      #- run: echo ${{ github.ref }}

From 2cb08223146134e477b8037772687e47aff7aabc Mon Sep 17 00:00:00 2001
From: Ian Davis <nospam@iandavis.com>
Date: Mon, 26 Jul 2021 10:52:28 +0100
Subject: [PATCH 09/34] Publish after nightly build

---
 .github/workflows/nightly.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index a24c0778..c25c15b1 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -25,4 +25,4 @@ jobs:
       - name: Connect to ipfs cluster
         run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
       - run: make deps nightly all_dists
-      #- run: make publish
+      - run: make publish

From f3c456df664abba7caf0907bb2d3a6140ebdee6a Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 26 Jul 2021 13:10:28 +0200
Subject: [PATCH 10/34] wip: wait for 2 cluster confirmations

---
 .github/workflows/main.yml | 126 +++++++++++++++++++++++--------------
 1 file changed, 80 insertions(+), 46 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index f8f26d1d..677afe15 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -23,7 +23,6 @@ jobs:
           node-version: '14'
       - name: Install ipfs and deps
         run: |
-          sudo snap install jq
           # set up go-ipfs
           IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
@@ -32,52 +31,63 @@ jobs:
           CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
           echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+      - name: Fix resolv
+        run: |
+          echo '-> flaky resolv'
+          ls -l /etc/resolv.conf
+          cat /etc/resolv.conf
+          echo '-> updated resolv'
+          # TODO: simplify
+          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
+          cat /etc/resolv.conf
       - name: Set up ipfs
         run: |
           ipfs init --profile server,test,lowpower
           # restore deterministic port (changed by test profile)
           ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
-          # DNS over HTTPS to avoid random Github Action failures like
-          # Error: lookup _dnsaddr.ipfs-websites.collab.ipfscluster.io on 127.0.0.53:53: dial udp 127.0.0.53:53: i/o timeout
-          ipfs config --json DNS.Resolvers '{ ".": "https://cloudflare-dns.com/dns-query" }'
       - name: Wait for ipfs daemon
         run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
         timeout-minutes: 3
-      - name: Connect to ipfs cluster
-        run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
-      - name: List peers
+      - name: Preconnect to cluster peers
         run: |
-          echo 'resolv'
-          ls -l /etc/resolv.conf
-          cat /etc/resolv.conf
-          echo 'updated resolv'
-          # TODO: simplify
-          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
-          cat /etc/resolv.conf
-          echo 'list swarm peers'
-          ipfs swarm peers
-          echo 'list cluster peers at ipfs-websites'
-          ipfs-cluster-ctl \
+          echo '-> preconnect to cluster peers'
+          ipfs-cluster-ctl --enc=json \
             --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
             --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
             peers ls > cluster-peers-ls
-          for peerid in $(grep "IPFS:" cluster-peers-ls | awk '{print $3; }'); do
-            for maddr in $(grep "/p2p/${peerid}" cluster-peers-ls | awk '{ print $2; }'); do
-              ipfs swarm connect "$maddr" || true
-            done
+          for maddr in $(jq -r '.[].ipfs.addresses[]' cluster-peers-ls); do
+            ipfs swarm connect "$maddr" || continue
           done
-          echo 'list swarm peers with cluster ones'
+          echo '-> manual connect to cluster.ipfs.io'
+          ipfs swarm connect /dnsaddr/cluster.ipfs.io
+          echo '-> list swarm peers'
           ipfs swarm peers
-      - name: Test pin
+        timeout-minutes: 3
+      - name: Pin test
         run: |
-          echo 'pin test cid'
-          ipfs-cluster-ctl \
+          TEST_CID=$(echo date | ipfs add -Q)
+          echo 'pin to cluster'
+          ipfs-cluster-ctl --enc=json \
             --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
             --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            --debug \
-            pin add QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR \
-            --wait --name  'test_${{ github.run_id }}' --expire-in 24h
-        timeout-minutes: 3
+            pin add \
+            --pin-name="test-ipfs-dists-${{ github.sha }}" \
+            --no-status \
+            "$TEST_CID"
+          while true; do
+            ipfs-cluster-ctl --enc=json \
+              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+              status "$TEST_CID" | tee cluster-pin-status
+            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
+              echo "Got 2 pin confirmations, finishing the workflow"
+              break
+            else
+              echo "(sleeping for 5 seconds)"
+              sleep 5
+            fi
+          done
+        timeout-minutes: 5
       - name: Build any new ./releases
         run: ./dockerized make all_dists
       - name: Inspect git status and contents of ./releases
@@ -240,7 +250,6 @@ jobs:
           node-version: '14'
       - name: Install ipfs and deps
         run: |
-          sudo snap install jq
           # set up go-ipfs
           IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
@@ -249,19 +258,38 @@ jobs:
           CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
           echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+      - name: Fix resolv
+        run: |
+          echo '-> flaky resolv'
+          ls -l /etc/resolv.conf
+          cat /etc/resolv.conf
+          echo '-> updated resolv'
+          # TODO: simplify
+          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
+          cat /etc/resolv.conf
       - name: Set up ipfs
         run: |
           ipfs init --profile server,test,lowpower
           # restore deterministic port (changed by test profile)
           ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
-          # DNS over HTTPS to avoid random Github Action failures like
-          # Error: lookup _dnsaddr.ipfs-websites.collab.ipfscluster.io on 127.0.0.53:53: dial udp 127.0.0.53:53: i/o timeout
-          ipfs config --json DNS.Resolvers '{ ".": "https://cloudflare-dns.com/dns-query" }'
       - name: Wait for ipfs daemon
         run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
         timeout-minutes: 3
-      - name: Connect to cluster.ipfs.io
-        run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
+      - name: Preconnect to cluster peers
+        run: |
+          echo '-> preconnect to cluster peers'
+          ipfs-cluster-ctl --enc=json \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            peers ls > cluster-peers-ls
+          for maddr in $(jq -r '.[].ipfs.addresses[]' cluster-peers-ls); do
+            ipfs swarm connect "$maddr" || continue
+          done
+          echo '-> manual connect to cluster.ipfs.io'
+          ipfs swarm connect /dnsaddr/cluster.ipfs.io
+          echo '-> list swarm peers'
+          ipfs swarm peers
+        timeout-minutes: 3
       - run: make publish
       - run: git status
       - name: Read CID of updated DAG
@@ -269,22 +297,28 @@ jobs:
         run: echo "::set-output name=CID::$(tail -1 ./versions)"
       - name: Pin new website to ipfs-websites.collab.ipfscluster.io
         run: |
-          ipfs swarm connect /dnsaddr/ipfs-websites.collab.ipfscluster.io
-          echo 'list swarm peers'
-          ipfs swarm peers
-          echo 'list cluster peers'
-          ipfs-cluster-ctl \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            peers ls
           echo 'pin to cluster'
-          ipfs-cluster-ctl \
+          ipfs-cluster-ctl --enc=json \
             --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
             --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
             pin add \
-            --wait --debug \
             --pin-name="https://github.com/ipfs/distributions/commits/${{ github.sha }}" \
+            --no-status \
             "${{ steps.cid-reader.outputs.CID }}"
+          while true; do
+            ipfs-cluster-ctl --enc=json \
+              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+              status "${{ steps.cid-reader.outputs.CID }}" | tee cluster-pin-status
+            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
+              echo "Got 2 pin confirmations, finishing the workflow"
+              break
+            else
+              echo "(sleeping for 5 seconds)"
+              sleep 5
+            fi
+          done
+        timeout-minutes: 60
       # TODO: update PR status with preview link
       # - uses: ipfs-shipyard/ipfs-github-action@51639f910548de983b3f49b209bc23e0cf997b14 # https://github.com/ipfs-shipyard/ipfs-github-action/pull/7
       #  id: ipfs

From 3e9a8e209801f443586a647654aaf5fe9e39ca16 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 26 Jul 2021 15:17:10 +0200
Subject: [PATCH 11/34] wip: speed up build

---
 .github/workflows/main.yml | 65 +++++++++-----------------------------
 Dockerfile                 |  2 +-
 2 files changed, 16 insertions(+), 51 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 677afe15..2ddbbdff 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -31,22 +31,17 @@ jobs:
           CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
           echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
-      - name: Fix resolv
-        run: |
-          echo '-> flaky resolv'
-          ls -l /etc/resolv.conf
-          cat /etc/resolv.conf
-          echo '-> updated resolv'
-          # TODO: simplify
-          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
-          cat /etc/resolv.conf
+      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
+        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
       - name: Set up ipfs
         run: |
-          ipfs init --profile server,test,lowpower
+          ipfs init --profile flatfs,server,test,lowpower
+          # make flatfs async for faster ci
+          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
           # restore deterministic port (changed by test profile)
           ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
       - name: Wait for ipfs daemon
-        run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
         timeout-minutes: 3
       - name: Preconnect to cluster peers
         run: |
@@ -63,31 +58,6 @@ jobs:
           echo '-> list swarm peers'
           ipfs swarm peers
         timeout-minutes: 3
-      - name: Pin test
-        run: |
-          TEST_CID=$(echo date | ipfs add -Q)
-          echo 'pin to cluster'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            pin add \
-            --pin-name="test-ipfs-dists-${{ github.sha }}" \
-            --no-status \
-            "$TEST_CID"
-          while true; do
-            ipfs-cluster-ctl --enc=json \
-              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-              status "$TEST_CID" | tee cluster-pin-status
-            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
-              echo "Got 2 pin confirmations, finishing the workflow"
-              break
-            else
-              echo "(sleeping for 5 seconds)"
-              sleep 5
-            fi
-          done
-        timeout-minutes: 5
       - name: Build any new ./releases
         run: ./dockerized make all_dists
       - name: Inspect git status and contents of ./releases
@@ -125,7 +95,7 @@ jobs:
         run: |
           brew tap mitchellh/gon
           brew install ipfs coreutils gawk gnu-sed jq mitchellh/gon/gon
-          ipfs init --profile server # needed for calculating NEW_CID later
+          ipfs init --profile test # needed for calculating NEW_CID later
       - name: Import Keychain Certs
         uses: apple-actions/import-codesign-certs@253ddeeac23f2bdad1646faac5c8c2832e800071 # v1@2020-02-03
         with:
@@ -258,22 +228,17 @@ jobs:
           CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
           tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
           echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
-      - name: Fix resolv
-        run: |
-          echo '-> flaky resolv'
-          ls -l /etc/resolv.conf
-          cat /etc/resolv.conf
-          echo '-> updated resolv'
-          # TODO: simplify
-          sudo sed -i -e 's/127.0.0.*/1.1.1.1/g' /etc/resolv.conf
-          cat /etc/resolv.conf
+      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
+        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
       - name: Set up ipfs
         run: |
-          ipfs init --profile server,test,lowpower
+          ipfs init --profile flatfs,server,test,lowpower
+          # make flatfs async for faster ci
+          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
           # restore deterministic port (changed by test profile)
           ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
       - name: Wait for ipfs daemon
-        run: ipfs daemon & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
         timeout-minutes: 3
       - name: Preconnect to cluster peers
         run: |
@@ -314,8 +279,8 @@ jobs:
               echo "Got 2 pin confirmations, finishing the workflow"
               break
             else
-              echo "(sleeping for 5 seconds)"
-              sleep 5
+              echo "(sleeping for 15 seconds)"
+              sleep 15
             fi
           done
         timeout-minutes: 60
diff --git a/Dockerfile b/Dockerfile
index ccd0bb86..604eb8dd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,7 @@
 FROM ubuntu:20.04
 ARG USER_UID
 RUN apt-get update -q && apt-get install -y git curl gnupg jq build-essential gawk zip 
-RUN curl -s https://dist.ipfs.io/go-ipfs/v0.8.0/go-ipfs_v0.8.0_linux-amd64.tar.gz | tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+RUN curl -s https://dist.ipfs.io/go-ipfs/v0.9.1/go-ipfs_v0.9.1_linux-amd64.tar.gz | tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
 
 RUN adduser --shell /bin/bash --home /asdf --disabled-password --gecos asdf asdf --uid $USER_UID
 ENV PATH="${PATH}:/asdf/.asdf/shims:/asdf/.asdf/bin"

From 796aaaef9b557ebe2cbb4234edba06418a6c6501 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 26 Jul 2021 19:38:32 +0200
Subject: [PATCH 12/34] wip: PR preview link

---
 .github/workflows/main.yml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 2ddbbdff..d6006fe8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -284,15 +284,16 @@ jobs:
             fi
           done
         timeout-minutes: 60
-      # TODO: update PR status with preview link
-      # - uses: ipfs-shipyard/ipfs-github-action@51639f910548de983b3f49b209bc23e0cf997b14 # https://github.com/ipfs-shipyard/ipfs-github-action/pull/7
-      #  id: ipfs
-      #  with:
-      #    path_to_add: /ipfs/${{ steps.cid-reader.outputs.CID }}
-      #    cluster_host: /dnsaddr/ipfs-websites.collab.ipfscluster.io
-      #    cluster_user: ${{ secrets.CLUSTER_USER }}
-      #    cluster_password: ${{ secrets.CLUSTER_PASSWORD }}
-      #  env:
-      #    GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      #- run: echo ${{ steps.ipfs.outputs.url }}
-      #- run: echo ${{ github.ref }}
+      - name: Update PR status with preview link
+        run: |
+          PREVIEW_URL="https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}"
+          API_PARAMS=$(jq --monochrome-output --null-input \
+            --arg state "success" \
+            --arg target_url "$PREVIEW_URL" \
+            --arg description "Preview on IPFS" \
+            --arg context "IPFS" \
+            '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
+          curl --output /dev/null --silent --show-error \
+            -X POST -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -H 'Content-Type: application/json' \
+            --data "$API_PARAMS" 'https://api.github.com/repos/ipfs/distributions/statuses/${{ github.sha }}'
+          echo "Pinned to IPFS - $PREVIEW_URL"

From ed656e5df46d5839e22959896cd80e968f9ff5fc Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Mon, 26 Jul 2021 23:19:54 +0200
Subject: [PATCH 13/34] wip: no-op build

this tests build without any new releases
---
 .github/workflows/main.yml     | 47 +++++++++++++---------------------
 README.md                      |  5 ++--
 deps-check.sh                  |  2 +-
 dists/ipfs-update/build_matrix | 12 +++++++++
 site/README.md                 |  2 +-
 5 files changed, 35 insertions(+), 33 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d6006fe8..be3f5584 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-# TODO: restore after testing
+# TODO: restore before merging
 #    branches:
 #      - master
 #  pull_request:
@@ -10,8 +10,9 @@ on:
 #      - master
 
 env:
- # TODO: remove after testing
-  DIST_ROOT: '/ipfs/QmX6J3hVtMF9Y73CKcyLHfgviEfXGFFQABWPXsD17EsBhg' # dist.ipfs.io without ipfs-update@v1.7.1
+ DIST_ROOT: '/ipns/dist.ipfs.io' # content root used for calculating diff to build
+ GO_IPFS_VER: 'v0.9.1'           # go-ipfs daemon used for chunking and applying diff
+ CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 
 jobs:
   build:
@@ -23,14 +24,8 @@ jobs:
           node-version: '14'
       - name: Install ipfs and deps
         run: |
-          # set up go-ipfs
-          IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
-          tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
-          echo "${{ github.workspace }}/go-ipfs/" >> $GITHUB_PATH
-          # set up ipfs-cluster-ctl
-          CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
-          tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
-          echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
       - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
         run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
       - name: Set up ipfs
@@ -50,7 +45,7 @@ jobs:
             --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
             --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
             peers ls > cluster-peers-ls
-          for maddr in $(jq -r '.[].ipfs.addresses[]' cluster-peers-ls); do
+          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
             ipfs swarm connect "$maddr" || continue
           done
           echo '-> manual connect to cluster.ipfs.io'
@@ -89,8 +84,9 @@ jobs:
         with:
           name: releases-unsigned-diff
           path: releases
+        continue-on-error: true # skip if no releases
       - name: List ./releases before
-        run: ls -Rhl ./releases
+        run: ls -Rhl ./releases || echo "No ./releases"
       - name: Install gon via HomeBrew for code signing and app notarization
         run: |
           brew tap mitchellh/gon
@@ -115,7 +111,7 @@ jobs:
               tar -zxvf "./releases/${DIST_NAME}/${DIST_VERSION}/${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz" -C "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/"
             done
           done
-          ls -Rhl ./tmp
+          ls -Rhl ./tmp || echo "Nothing new in ./tmp"
       - name: Sign and notarize the mac binaries
         env:
           AC_USERNAME: ${{ secrets.APPLE_AC_USERNAME }} # implicitly read from env by gon
@@ -195,13 +191,14 @@ jobs:
             done
           done
       - name: List ./releases after
-        run: ls -Rhl ./releases
+        run: ls -Rhl ./releases || echo "No ./releases"
       - name: Temporarily save notarized artifacts
         uses: actions/upload-artifact@v2
         with:
           name: releases-signed-macos-diff
           path: releases
           retention-days: 1
+        continue-on-error: true # skip if no releases
 
   persist:
     runs-on: "ubuntu-latest"
@@ -210,24 +207,16 @@ jobs:
       - uses: actions/checkout@v2
       - name: Retrieve signed artifacts
         uses: actions/download-artifact@v2
+        continue-on-error: true # skip if no releases
         with:
           name: releases-signed-macos-diff
           path: releases
       - name: List ./releases
-        run: ls -Rhl ./releases
-      - uses: actions/setup-node@v2
-        with:
-          node-version: '14'
+        run: ls -Rhl ./releases || echo "No ./releases"
       - name: Install ipfs and deps
         run: |
-          # set up go-ipfs
-          IPFS_VERSION=$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)
-          tar xzf <(curl -s https://dist.ipfs.io/go-ipfs/${IPFS_VERSION}/go-ipfs_${IPFS_VERSION}_linux-amd64.tar.gz)
-          echo "${{ github.workspace }}/go-ipfs/" >> $GITHUB_PATH
-          # set up ipfs-cluster-ctl
-          CTL_VERSION=$(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/versions | tail -n 1)
-          tar xzf <(curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CTL_VERSION}/ipfs-cluster-ctl_${CTL_VERSION}_linux-amd64.tar.gz)
-          echo "${{ github.workspace }}/ipfs-cluster-ctl/" >> $GITHUB_PATH
+          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
       - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
         run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
       - name: Set up ipfs
@@ -247,7 +236,7 @@ jobs:
             --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
             --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
             peers ls > cluster-peers-ls
-          for maddr in $(jq -r '.[].ipfs.addresses[]' cluster-peers-ls); do
+          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
             ipfs swarm connect "$maddr" || continue
           done
           echo '-> manual connect to cluster.ipfs.io'
@@ -255,7 +244,7 @@ jobs:
           echo '-> list swarm peers'
           ipfs swarm peers
         timeout-minutes: 3
-      - run: make publish
+      - run: ./dockerized make publish
       - run: git status
       - name: Read CID of updated DAG
         id: cid-reader
diff --git a/README.md b/README.md
index a438f817..39c5a9d7 100644
--- a/README.md
+++ b/README.md
@@ -122,14 +122,15 @@ To produce a CID (`<NEW_HASH>`) that includes binaries for all versions defined
 
 After the local build is done, make a quick inspection:
 
-2. Load the dists website in your browser to make sure everything looks right: `http://127.0.0.1:8080/ipfs/<NEW_HASH>`.
+2. Load the dists website in your browser to make sure everything looks right: `http://localhost:8080/ipfs/<NEW_HASH>`.
 3. Compare `<NEW_HASH>` with the current `dists.ipfs.io` to make sure nothing is amiss: `ipfs object diff /ipns/dist.ipfs.io /ipfs/<NEW_HASH>`
 
 Finally,
 
 1. Commit your changes and make a PR. Specifically, the changes to `dists/<dist>/versions` and `dists/<dist>/current`.
-2. Wait for [Github Action](https://github.com/ipfs/distributions/actions/) on your PR to build and **signed** binaries. `<NEW_SIGNED_HASH>` will be different than one from local build.
+2. Wait for [Github Action](https://github.com/ipfs/distributions/actions/) on your PR to build **signed** binaries. `<NEW_SIGNED_HASH>` will be different than one from local build.
 3. Make a PR with an edit on [protocol/infra](https://github.com/protocol/infra/blob/master/dns/config/dist.ipfs.io.yaml) with `<NEW_SIGNED_HASH>` you got from the Github Action output and a link to the PR above.
+   - TODO: this step will be automated in the future.
 
 If you have permission, you can just merge the PR, update the DNS, and then immediately, close the issue on ipfs/infrastructure. Ping someone on IRC.
 
diff --git a/deps-check.sh b/deps-check.sh
index edb56df2..1a64d719 100755
--- a/deps-check.sh
+++ b/deps-check.sh
@@ -25,4 +25,4 @@ if [ "$failed" = true ]; then
 fi
 
 echo "npm install"
-exec npm install --no-audit --progress=false
+exec npm ci --prefer-offline --no-audit --progress=false
diff --git a/dists/ipfs-update/build_matrix b/dists/ipfs-update/build_matrix
index a523aa54..c562e61b 100644
--- a/dists/ipfs-update/build_matrix
+++ b/dists/ipfs-update/build_matrix
@@ -1,2 +1,14 @@
 darwin amd64
 darwin arm64
+freebsd 386
+freebsd amd64
+freebsd arm
+openbsd 386
+openbsd amd64
+openbsd arm
+linux 386
+linux amd64
+linux arm
+linux arm64
+windows 386
+windows amd64
diff --git a/site/README.md b/site/README.md
index 21db74ac..fcef7453 100644
--- a/site/README.md
+++ b/site/README.md
@@ -6,7 +6,7 @@
 
 ```bash
 # Install dependencies
-$ npm install
+$ npm ci
 $ npm start
 $ open localhost:1313
 ```

From 7d222b592f5c97df0d3be2e4c997598fc06e6e3c Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 13:43:02 +0200
Subject: [PATCH 14/34] refactor: use GO_IPFS_VER in dockerized build

---
 Dockerfile | 3 ++-
 dockerized | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 604eb8dd..11cdb537 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,7 +1,8 @@
 FROM ubuntu:20.04
 ARG USER_UID
+ARG GO_IPFS_VER
 RUN apt-get update -q && apt-get install -y git curl gnupg jq build-essential gawk zip 
-RUN curl -s https://dist.ipfs.io/go-ipfs/v0.9.1/go-ipfs_v0.9.1_linux-amd64.tar.gz | tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+RUN curl -s "https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz" | tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
 
 RUN adduser --shell /bin/bash --home /asdf --disabled-password --gecos asdf asdf --uid $USER_UID
 ENV PATH="${PATH}:/asdf/.asdf/shims:/asdf/.asdf/bin"
diff --git a/dockerized b/dockerized
index d5f15c85..5e248541 100755
--- a/dockerized
+++ b/dockerized
@@ -5,7 +5,10 @@ set -euxo pipefail
 docker pull ubuntu:20.04
 
 # CACHEBUST means this will apply the updates once a day
-docker build . -t distributions --build-arg CACHEBUST=`date --iso-8601=date` --build-arg USER_UID=$(id -u "$USER")
+docker build . -t distributions \
+    --build-arg CACHEBUST=`date --iso-8601=date` \
+    --build-arg USER_UID=$(id -u "$USER") \
+    --build-arg GO_IPFS_VER=${GO_IPFS_VER:-$(curl -s https://dist.ipfs.io/go-ipfs/versions | tail -n 1)} # match http api client version on CI
 
 # We use host networking as the build process assumes a fairly long-lived ipfs
 # node has the CIDs (we give them to the collab cluster to pin) 

From f115f93a8dcd56651da55792f27d7624e6e8823f Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 13:47:17 +0200
Subject: [PATCH 15/34] docs: DNSLink discussion

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 39c5a9d7..63b44e3a 100644
--- a/README.md
+++ b/README.md
@@ -130,7 +130,7 @@ Finally,
 1. Commit your changes and make a PR. Specifically, the changes to `dists/<dist>/versions` and `dists/<dist>/current`.
 2. Wait for [Github Action](https://github.com/ipfs/distributions/actions/) on your PR to build **signed** binaries. `<NEW_SIGNED_HASH>` will be different than one from local build.
 3. Make a PR with an edit on [protocol/infra](https://github.com/protocol/infra/blob/master/dns/config/dist.ipfs.io.yaml) with `<NEW_SIGNED_HASH>` you got from the Github Action output and a link to the PR above.
-   - TODO: this step will be automated in the future.
+   - TODO: this step may be automated in the future - see the [discussion](https://github.com/ipfs/distributions/issues/372).
 
 If you have permission, you can just merge the PR, update the DNS, and then immediately, close the issue on ipfs/infrastructure. Ping someone on IRC.
 

From 9acff7e8feecb8520009c2a86e50935161c5c992 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 14:13:27 +0200
Subject: [PATCH 16/34] refactor: simplify

https://github.com/ipfs/distributions/pull/367#discussion_r677004732
---
 build-go.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index 8e0a6cf2..15ccd1ae 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -10,8 +10,7 @@ export GOPATH
 export GO111MODULE=on
 
 # Content path to use when looking for pre-existing release data
-DIST_ROOT=${DIST_ROOT:-/ipns/dist.ipfs.io}
-DIST_ROOT=$(ipfs resolve "$DIST_ROOT")
+DIST_ROOT=$(ipfs resolve "${DIST_ROOT:-/ipns/dist.ipfs.io}")
 
 # normalize umask
 umask 022

From 5ea24dc526f526d1a2251573a6dc1757758897bf Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 17:14:56 +0200
Subject: [PATCH 17/34] refactor: avoid polluting ./releases

This is a small refactor that makes sure we only add meaningful files to
./releases dir (it is important for signing etc)
---
 build-go.sh | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index 9b90eb5a..c1a3000e 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -349,19 +349,10 @@ function startGoBuilds() {
 	fi
 
 	echo "comparing $versions with $existing/$distname/versions"
+	existingVersions=$(mktemp)
+	ipfs cat "$existing/$distname/versions" > "$existingVersions"
 
-	outputDir="$releases/$distname"
-	# if the output directory already exists, warn user
-	if [ -e "$outputDir" ]; then
-		warn "dirty output directory"
-		warn "will skip building already existing binaries"
-		warn "to perform a fresh build, please delete $outputDir"
-	fi
-
-	mkdir -p "$outputDir"
-	touch "$outputDir/existingVersions"
-	ipfs cat "$existing/$distname/versions" >> "$outputDir/existingVersions"
-	newVersions=$(comm --nocheck-order -13 "$outputDir/existingVersions" "$versions")
+	newVersions=$(comm --nocheck-order -13 "$existingVersions" "$versions")
 
 	if [ -z "$newVersions" ]; then
 		notice "skipping $distname - all versions published at $existing"
@@ -369,7 +360,14 @@ function startGoBuilds() {
 	fi
 	printVersions "$newVersions"
 
+	outputDir="$releases/$distname"
 
+	# if the output directory already exists, warn user
+	if [ -e "$outputDir" ]; then
+		warn "dirty output directory"
+		warn "will skip building already existing binaries"
+		warn "to perform a fresh build, please delete $outputDir"
+	fi
 
 	export GOPATH
 	GOPATH="$(pwd)/gopath"
@@ -425,13 +423,15 @@ function startGoBuilds() {
 		echo ""
 	done <<< "$newVersions"
 
-	# All tagged versions from repo
+	# Keep all tagged versions from repo
 	grep -v ^nightly "$versions" > "$outputDir/versions"
 
-	# Last 7 nightly versions from repo and existing
-	grep -h ^nightly "$versions" "$outputDir/existingVersions" | sort -ur | head -n7 >> "$outputDir/versions"
+	# Keep at most 7 nightly versions
+	grep -h ^nightly "$versions" "$existingVersions" | sort -ur | head -n7 >> "$outputDir/versions"
 
 	notice "build complete!"
 }
 
 startGoBuilds "$1" "$2" "$3" "$4" "$5"
+
+# vim: noet

From 5df807b8531256d31672b4b31906a38a5775e977 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 19:08:12 +0200
Subject: [PATCH 18/34] refactor: respect nightly cut-off date
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This ensures that "nightly-2020-01-01" won't have any commits that
happened on "2020-01-02" – enabling us to build historical nightlies.
---
 build-go.sh | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index c1a3000e..f5c3c03e 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -251,17 +251,26 @@ function cleanRepo() {
 	git -C "$repopath" reset --hard
 }
 
+function nightlyRevision() {
+	local repopath=$1
+	local version=$2
+
+	# Use default branch, may be master, main or some other name
+	default_branch="$(git -C "$repopath" symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')"
+	# Find the last commit before nightly cut-off
+	cutoff_date="${version#nightly-}"
+	git -C "$repopath" rev-list -1 --first-parent --before="$cutoff_date" "$default_branch"
+}
+
 function checkoutVersion() {
 	local repopath=$1
 	local version=$2
 
 	test -n "$repopath" || fail "checkoutVersion: no repo to check out specified"
 
-
 	case $version in
 	nightly*)
-		# Use default branch, may be master, main or some other name
-		ref=$(git -C "$repopath" symbolic-ref refs/remotes/origin/HEAD | sed 's@^refs/remotes/origin/@@')
+		ref="$(nightlyRevision "$repopath" "$version")"
 		;;
 	*)
 		ref=$version
@@ -392,7 +401,7 @@ function startGoBuilds() {
 
 		case $version in
 		nightly*)
-			buildVersion="$version-$(git rev-parse --short=7 HEAD)"
+			buildVersion="$version-$(nightlyRevision "$repopath" "$version" | head -c 7)"
 			;;
 		*)
 			buildVersion=$version

From 716ebafdcca3956ba20d3ff3ffe1c82264365183 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 21:19:16 +0200
Subject: [PATCH 19/34] wip: see how long all nightlies take to build and pin

---
 .github/workflows/nightly.yml | 100 ++++++++++++++++++++++++++++------
 1 file changed, 83 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index c25c15b1..ae2e6794 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -1,28 +1,94 @@
 name: Nightly
 
 on:
+  push:
   ## Allow manual invocation
-  workflow_dispatch:
-  schedule:
-    - cron: '0 5 * * *' # UTC
+  #workflow_dispatch:
+  #schedule:
+  #  - cron: '0 5 * * *' # UTC
+
+env:
+ DIST_ROOT: '/ipns/dist.ipfs.io' # content root used for calculating diff to build
+ GO_IPFS_VER: 'v0.9.1'           # go-ipfs daemon used for chunking and applying diff
+ CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 
 jobs:
   build:
     runs-on: "ubuntu-latest"
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-node@v2-beta
-        with:
-          node-version: '14'
-      - uses: actions/setup-go@v2
-        with:
-          go-version: '1.16'
-      - run: sudo snap install ipfs jq
-      - run: ipfs init --profile server
-      - run: ipfs daemon &
+      - name: Install ipfs and deps
+        run: |
+          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
+      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
+        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
+      - name: Set up ipfs
+        run: |
+          ipfs init --profile flatfs,server,test,lowpower
+          # make flatfs async for faster ci
+          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
+          # restore deterministic port (changed by test profile)
+          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
       - name: Wait for ipfs daemon
-        run: npx wait-port http://127.0.0.1:8080/api/v0/version
-      - name: Connect to ipfs cluster
-        run: ipfs swarm connect /dnsaddr/cluster.ipfs.io
-      - run: make deps nightly all_dists
-      - run: make publish
+        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+        timeout-minutes: 3
+      - name: Preconnect to cluster peers
+        run: |
+          echo '-> preconnect to cluster peers'
+          ipfs-cluster-ctl --enc=json \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            peers ls > cluster-peers-ls
+          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
+            ipfs swarm connect "$maddr" || continue
+          done
+          echo '-> manual connect to cluster.ipfs.io'
+          ipfs swarm connect /dnsaddr/cluster.ipfs.io
+          echo '-> list swarm peers'
+          ipfs swarm peers
+        timeout-minutes: 3
+      - run: ./dockerized make deps nightly all_dists
+      - run: ./dockerized make publish
+      - name: Inspect git status and contents of ./releases
+        run: git status && ls -Rhl ./releases
+      - name: Read CID of updated DAG
+        id: cid-reader
+        run: echo "::set-output name=CID::$(tail -1 ./versions)"
+      - name: Pin new website to ipfs-websites.collab.ipfscluster.io for 7 days
+        run: |
+          echo 'pin to cluster'
+          ipfs-cluster-ctl --enc=json \
+            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+            pin add \
+            --pin-name="https://github.com/ipfs/distributions/commits/${{ github.sha }}" \
+            --no-status --expire-in 168h \
+            "${{ steps.cid-reader.outputs.CID }}"
+          while true; do
+            ipfs-cluster-ctl --enc=json \
+              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
+              status "${{ steps.cid-reader.outputs.CID }}" | tee cluster-pin-status
+            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
+              echo "Got 2 pin confirmations, finishing the workflow"
+              break
+            else
+              echo "(sleeping for 15 seconds)"
+              sleep 15
+            fi
+          done
+        timeout-minutes: 60
+      - name: Update PR status with preview link
+        run: |
+          PREVIEW_URL="https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}"
+          API_PARAMS=$(jq --monochrome-output --null-input \
+            --arg state "success" \
+            --arg target_url "$PREVIEW_URL" \
+            --arg description "Preview on IPFS" \
+            --arg context "IPFS" \
+            '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
+          curl --output /dev/null --silent --show-error \
+            -X POST -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -H 'Content-Type: application/json' \
+            --data "$API_PARAMS" 'https://api.github.com/repos/ipfs/distributions/statuses/${{ github.sha }}'
+          echo "Pinned to IPFS - $PREVIEW_URL"

From 6e513af8f23ff6d0903ea1d42fb2e2f7b0d6373b Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Tue, 27 Jul 2021 23:23:20 +0200
Subject: [PATCH 20/34] fix: dockerized build on ci

---
 dockerized | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dockerized b/dockerized
index 80c296cc..1671335e 100755
--- a/dockerized
+++ b/dockerized
@@ -9,4 +9,4 @@ docker build . -t distributions --build-arg CACHEBUST=`date --iso-8601=date` --b
 
 # We use host networking as the build process assumes a fairly long-lived ipfs
 # node has the CIDs (we give them to the collab cluster to pin) 
-docker run --rm -it --network host -v `pwd`:/build distributions "$@"
+docker run --rm -i --network host -v `pwd`:/build distributions "$@"

From baa201f922d20d92fbc55778219880660959ce05 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 02:26:22 +0200
Subject: [PATCH 21/34] fix: vtag dists like fs-repo-migrations/fs-repo-*

This makes sure we use git tag in 'vtag' style for regular releases
and not for nightlies (which use raw git revision based on date cutoff)
---
 build-go.sh | 20 ++++++++++++--------
 1 file changed, 12 insertions(+), 8 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index f5c3c03e..94d6a457 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -199,7 +199,7 @@ function printInitialDistfile() {
   "name": "$distname",
   "owner": "$(< repo-owner)",
   "description": "$(< description)",
-  "date": "$(date '+%B %d, %Y')",
+  "date": "$(date -u '+%B %d, %Y')",
   "platforms": {}
 }
 EOF
@@ -211,7 +211,7 @@ function printBuildInfo() {
 	go version
 	echo "git sha of code: $commit"
 	uname -a
-	echo "built on $(date)"
+	echo "built on $(date -u)"
 }
 
 function buildWithMatrix() {
@@ -274,17 +274,20 @@ function checkoutVersion() {
 		;;
 	*)
 		ref=$version
+
+		# If there is a vtag, then checkout using <vtag>/<version>
+		# ('vtag' file is used for indicating 'submodule'
+		# such as 'fs-repo-migrations/fs-repo-0-to-1',
+		# those have release tags like 'fs-repo-0-to-1/v1.0.0')
+		if [ -e vtag ]; then
+			ref="$(cat vtag)/${version}"
+		fi
 		;;
 	esac
 
-	echo "==> checking out version $ref in $repopath"
+	echo "==> checking out version $version (git: $ref) in $repopath"
 	cleanRepo "$repopath"
 
-	# If there is a vtag, then checkout using <vtag>/<version>
-	if [ -e vtag ]; then
-		ref="$(cat vtag)/${ref}"
-	fi
-
 	git -C "$repopath" checkout "$ref" > /dev/null || fail "failed to check out $ref in $reporoot"
 }
 
@@ -358,6 +361,7 @@ function startGoBuilds() {
 	fi
 
 	echo "comparing $versions with $existing/$distname/versions"
+
 	existingVersions=$(mktemp)
 	ipfs cat "$existing/$distname/versions" > "$existingVersions"
 

From a2e8f266e365fa0c3cf3d40441b5d969863e8404 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 03:09:41 +0200
Subject: [PATCH 22/34] wip: run each dist in matrix

this means we see which is green and which fails,
we can merge them together at the end but I feel its already super
useful as-is
---
 .github/workflows/nightly.yml | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index ae2e6794..6f000dbb 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -13,8 +13,22 @@ env:
  CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 
 jobs:
+  prepare-matrix:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(jq -nc '$ARGS.positional' --args $(ls ./dists -1))"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
   build:
     runs-on: "ubuntu-latest"
+    needs: prepare-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        dist_name: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v2
       - name: Install ipfs and deps
@@ -48,7 +62,7 @@ jobs:
           echo '-> list swarm peers'
           ipfs swarm peers
         timeout-minutes: 3
-      - run: ./dockerized make deps nightly all_dists
+      - run: cd ./dists/${{ matrix.dist_name }} && make nightly
       - run: ./dockerized make publish
       - name: Inspect git status and contents of ./releases
         run: git status && ls -Rhl ./releases
@@ -72,6 +86,7 @@ jobs:
               status "${{ steps.cid-reader.outputs.CID }}" | tee cluster-pin-status
             if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
               echo "Got 2 pin confirmations, finishing the workflow"
+              echo "Pinned ${{ matrix.dist_name }} nightly to IPFS - https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}/${{ matrix.dist_name }}/"
               break
             else
               echo "(sleeping for 15 seconds)"
@@ -79,16 +94,3 @@ jobs:
             fi
           done
         timeout-minutes: 60
-      - name: Update PR status with preview link
-        run: |
-          PREVIEW_URL="https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}"
-          API_PARAMS=$(jq --monochrome-output --null-input \
-            --arg state "success" \
-            --arg target_url "$PREVIEW_URL" \
-            --arg description "Preview on IPFS" \
-            --arg context "IPFS" \
-            '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
-          curl --output /dev/null --silent --show-error \
-            -X POST -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -H 'Content-Type: application/json' \
-            --data "$API_PARAMS" 'https://api.github.com/repos/ipfs/distributions/statuses/${{ github.sha }}'
-          echo "Pinned to IPFS - $PREVIEW_URL"

From 87c0b1130276a1b3b8718fb364cccaeb040cbd42 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 15:03:10 +0200
Subject: [PATCH 23/34] refactor: scripts/ci/setup-ipfs.sh

---
 .github/workflows/main.yml | 74 +++++++-------------------------------
 scripts/ci/setup-ipfs.sh   | 37 +++++++++++++++++++
 2 files changed, 49 insertions(+), 62 deletions(-)
 create mode 100644 scripts/ci/setup-ipfs.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index be3f5584..d9bfba92 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -22,37 +22,12 @@ jobs:
       - uses: actions/setup-node@v2
         with:
           node-version: '14'
-      - name: Install ipfs and deps
-        run: |
-          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
-          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
-      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
-        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
-      - name: Set up ipfs
-        run: |
-          ipfs init --profile flatfs,server,test,lowpower
-          # make flatfs async for faster ci
-          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
-          # restore deterministic port (changed by test profile)
-          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
-      - name: Wait for ipfs daemon
-        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
-        timeout-minutes: 3
-      - name: Preconnect to cluster peers
-        run: |
-          echo '-> preconnect to cluster peers'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            peers ls > cluster-peers-ls
-          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
-            ipfs swarm connect "$maddr" || continue
-          done
-          echo '-> manual connect to cluster.ipfs.io'
-          ipfs swarm connect /dnsaddr/cluster.ipfs.io
-          echo '-> list swarm peers'
-          ipfs swarm peers
-        timeout-minutes: 3
+      - name: Setup IPFS
+        run: ./scripts/ci/setup-ipfs.sh
+        env:
+          CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
+          CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
+        timeout-minutes: 5
       - name: Build any new ./releases
         run: ./dockerized make all_dists
       - name: Inspect git status and contents of ./releases
@@ -213,37 +188,12 @@ jobs:
           path: releases
       - name: List ./releases
         run: ls -Rhl ./releases || echo "No ./releases"
-      - name: Install ipfs and deps
-        run: |
-          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
-          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
-      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
-        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
-      - name: Set up ipfs
-        run: |
-          ipfs init --profile flatfs,server,test,lowpower
-          # make flatfs async for faster ci
-          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
-          # restore deterministic port (changed by test profile)
-          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
-      - name: Wait for ipfs daemon
-        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
-        timeout-minutes: 3
-      - name: Preconnect to cluster peers
-        run: |
-          echo '-> preconnect to cluster peers'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            peers ls > cluster-peers-ls
-          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
-            ipfs swarm connect "$maddr" || continue
-          done
-          echo '-> manual connect to cluster.ipfs.io'
-          ipfs swarm connect /dnsaddr/cluster.ipfs.io
-          echo '-> list swarm peers'
-          ipfs swarm peers
-        timeout-minutes: 3
+      - name: Setup IPFS
+        run: ./scripts/ci/setup-ipfs.sh
+        env:
+          CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
+          CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
+        timeout-minutes: 5
       - run: ./dockerized make publish
       - run: git status
       - name: Read CID of updated DAG
diff --git a/scripts/ci/setup-ipfs.sh b/scripts/ci/setup-ipfs.sh
new file mode 100644
index 00000000..18b379a2
--- /dev/null
+++ b/scripts/ci/setup-ipfs.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+set -e
+
+echo "::group::Install go-ipfs and ipfs-cluster-ctl"
+    curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
+    curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
+echo "::endgroup::"
+
+# fix resolv - DNS provided by Github is unreliable for DNSLik/dnsaddr
+sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
+
+# init ipfs
+echo "::group::Set up IPFS daemon"
+    ipfs init --profile flatfs,server,test,lowpower
+    # make flatfs async for faster ci
+    new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
+    # restore deterministic port (changed by test profile)
+    ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
+    # wait for ipfs daemon
+    ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
+echo "::endgroup::"
+
+
+echo "::group::Preconnect to cluster peers"
+    echo '-> preconnect to cluster peers'
+    ipfs-cluster-ctl --enc=json \
+        --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+        --basic-auth '${CLUSTER_USER}:${CLUSTER_PASSWORD}' \
+        peers ls > cluster-peers-ls
+    for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
+        ipfs swarm connect "$maddr" || continue
+    done
+    echo '-> manual connect to cluster.ipfs.io'
+    ipfs swarm connect /dnsaddr/cluster.ipfs.io
+    echo '-> list swarm peers'
+    ipfs swarm peers
+echo "::endgroup::"

From 954465432dfcea153ec8ba0a099ee144e24b7fae Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 15:13:02 +0200
Subject: [PATCH 24/34] refactor: scripts/ci/pin-to-cluster.sh

---
 .github/workflows/main.yml   | 29 +++++++----------------------
 scripts/ci/pin-to-cluster.sh | 28 ++++++++++++++++++++++++++++
 scripts/ci/setup-ipfs.sh     |  2 +-
 3 files changed, 36 insertions(+), 23 deletions(-)
 create mode 100755 scripts/ci/pin-to-cluster.sh
 mode change 100644 => 100755 scripts/ci/setup-ipfs.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d9bfba92..c3a5dffe 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -200,28 +200,13 @@ jobs:
         id: cid-reader
         run: echo "::set-output name=CID::$(tail -1 ./versions)"
       - name: Pin new website to ipfs-websites.collab.ipfscluster.io
-        run: |
-          echo 'pin to cluster'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            pin add \
-            --pin-name="https://github.com/ipfs/distributions/commits/${{ github.sha }}" \
-            --no-status \
-            "${{ steps.cid-reader.outputs.CID }}"
-          while true; do
-            ipfs-cluster-ctl --enc=json \
-              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-              status "${{ steps.cid-reader.outputs.CID }}" | tee cluster-pin-status
-            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
-              echo "Got 2 pin confirmations, finishing the workflow"
-              break
-            else
-              echo "(sleeping for 15 seconds)"
-              sleep 15
-            fi
-          done
+        run: ./scripts/ci/pin-to-cluster.sh
+        env:
+          PIN_CID: ${{ steps.cid-reader.outputs.CID }}
+          PIN_NAME: "https://github.com/ipfs/distributions/commits/${{ github.sha }}"
+          PIN_ADD_EXTRA_ARGS: ""
+          CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
+          CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
         timeout-minutes: 60
       - name: Update PR status with preview link
         run: |
diff --git a/scripts/ci/pin-to-cluster.sh b/scripts/ci/pin-to-cluster.sh
new file mode 100755
index 00000000..d4f5527b
--- /dev/null
+++ b/scripts/ci/pin-to-cluster.sh
@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+set -e
+
+echo "::group::pin add"
+ipfs-cluster-ctl --enc=json \
+    --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+    --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
+    pin add \
+    --pin-name="${PIN_NAME}" \
+    --no-status $PIN_ADD_EXTRA_ARGS \
+    "PIN_CID"
+echo "::endgroup::"
+
+echo "::group::waiting until pinned"
+    while true; do
+    ipfs-cluster-ctl --enc=json \
+        --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
+        --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
+        status "PIN_CID" | tee cluster-pin-status
+    if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
+        echo "Got 2 pin confirmations, finishing the workflow"
+        break
+    else
+        echo "(sleeping for 15 seconds)"
+        sleep 15
+    fi
+    done
+echo "::endgroup::"
diff --git a/scripts/ci/setup-ipfs.sh b/scripts/ci/setup-ipfs.sh
old mode 100644
new mode 100755
index 18b379a2..d3e41163
--- a/scripts/ci/setup-ipfs.sh
+++ b/scripts/ci/setup-ipfs.sh
@@ -25,7 +25,7 @@ echo "::group::Preconnect to cluster peers"
     echo '-> preconnect to cluster peers'
     ipfs-cluster-ctl --enc=json \
         --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-        --basic-auth '${CLUSTER_USER}:${CLUSTER_PASSWORD}' \
+        --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
         peers ls > cluster-peers-ls
     for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
         ipfs swarm connect "$maddr" || continue

From c239cfbbbff24e1235b6ebc1183d0b41d392ef0f Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 15:17:18 +0200
Subject: [PATCH 25/34] fix: quic perf

---
 scripts/ci/setup-ipfs.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/scripts/ci/setup-ipfs.sh b/scripts/ci/setup-ipfs.sh
index d3e41163..2479f2ed 100755
--- a/scripts/ci/setup-ipfs.sh
+++ b/scripts/ci/setup-ipfs.sh
@@ -9,6 +9,9 @@ echo "::endgroup::"
 # fix resolv - DNS provided by Github is unreliable for DNSLik/dnsaddr
 sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
 
+# QUIC perf: https://github.com/lucas-clemente/quic-go/wiki/UDP-Receive-Buffer-Size
+sudo sysctl -w net.core.rmem_max=2500000
+
 # init ipfs
 echo "::group::Set up IPFS daemon"
     ipfs init --profile flatfs,server,test,lowpower

From ef7843a2a658c0dc062f9a0b1bee2d580a00ab2c Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 15:32:48 +0200
Subject: [PATCH 26/34] refactor: sign-new-macos-releases.sh

---
 .github/workflows/main.yml            | 91 +-------------------------
 scripts/ci/pin-to-cluster.sh          |  4 +-
 scripts/ci/sign-new-macos-releases.sh | 92 +++++++++++++++++++++++++++
 3 files changed, 97 insertions(+), 90 deletions(-)
 create mode 100755 scripts/ci/sign-new-macos-releases.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index c3a5dffe..b0a0ed6e 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -74,97 +74,12 @@ jobs:
           p12-password: ${{ secrets.APPLE_CERTS_PASS }}
       - name: Verify identity used for signing
         run: security find-identity -v
-      - name: Unpack any new darwin arm64 and amd64 binaries to ./tmp
-        run: |
-          # ./releases/{DIST_NAME}/{DIST_VERSION}/*_darwin-${arch}.tar.gz -> ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
-          for NEW_DIR in ./releases/*/v*; do
-            (! test -d "$NEW_DIR") && continue
-            DIST_VERSION=$(basename "$NEW_DIR")
-            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
-            for arch in "amd64" "arm64"; do
-              mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned"
-              tar -zxvf "./releases/${DIST_NAME}/${DIST_VERSION}/${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz" -C "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/"
-            done
-          done
-          ls -Rhl ./tmp || echo "Nothing new in ./tmp"
-      - name: Sign and notarize the mac binaries
+      - name: Sign any new releases
+        run: ./scripts/ci/sign-new-macos-releases.sh
         env:
+          WORK_DIR: ${{ github.workspace }}
           AC_USERNAME: ${{ secrets.APPLE_AC_USERNAME }} # implicitly read from env by gon
           AC_PASSWORD: ${{ secrets.APPLE_AC_PASSWORD }}
-        run: |
-          # Find and sign executables in ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
-          for NEW_DIR in ./releases/*/v*; do
-            (! test -d "$NEW_DIR") && continue
-            DIST_VERSION=$(basename "$NEW_DIR")
-            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
-            for arch in "amd64" "arm64"; do
-              EXECUTABLES=$(jq -nc '$ARGS.positional' --args $(find "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/" -perm +111 -type f -print))
-              echo "{
-                  \"source\" : $EXECUTABLES,
-                  \"bundle_id\" : \"io.ipfs.dist.${DIST_NAME}\",
-                  \"apple_id\": {
-                    \"password\":  \"@env:AC_PASSWORD\"
-                  },
-                  \"sign\" :{
-                    \"application_identity\" : \"Developer ID Application: Protocol Labs, Inc. (7Y229E2YRL)\"
-                  },
-                  \"zip\" :{
-                      \"output_path\" : \"./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip\"
-                  }
-              }" | tee | jq > "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
-              gon -log-level=info -log-json "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
-            done
-          done
-      - name: Temporarily save ./tmp
-        uses: actions/upload-artifact@v2
-        with:
-          name: tmp
-          path: ./tmp/
-          retention-days: 1
-      - name: Update changed binaries in ./releases
-        run: |
-          for NEW_DIR in ./releases/*/v*; do
-            (! test -d "$NEW_DIR") && continue
-            DIST_VERSION=$(basename "$NEW_DIR")
-            DIST_NAME=$(basename $(dirname "$NEW_DIR"))
-            for arch in "amd64" "arm64"; do
-              echo "-> Starting the update of darwin_${arch}.tar.gz for name='${DIST_NAME}' and version='${DIST_VERSION}'"
-              # unzip signed binaries to a directory matching .tar.gz structure
-              cd "${{ github.workspace }}"
-              mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}"
-              cd "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}/"
-              echo "-> Unpacking gon .zip for ${arch}"
-              unzip "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip"
-              echo "-> Unpacked contents"
-              ls -Rhl "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/"
-              # replace .tar.gz with one that has the same structure, but signed binaries
-              PKG_NAME="${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz"
-              PKG_ROOT="${{ github.workspace }}/releases/${DIST_NAME}/${DIST_VERSION}"
-              PKG_PATH="${PKG_ROOT}/${PKG_NAME}"
-              DIST_JSON="${PKG_ROOT}/dist.json"
-              # read old hashes
-              OLD_CID=$(cat "${PKG_PATH}.cid")
-              OLD_SHA512=$(gawk '{ print $1; }' < "${PKG_PATH}.sha512")
-              echo "-> Found old $PKG_NAME"
-              echo "   old CID:    $OLD_CID"
-              echo "   old SHA512: $OLD_SHA512"
-              echo "-> Updating $PKG_NAME"
-              rm "$PKG_PATH"
-              tar -czvf "${{ github.workspace }}/releases/${DIST_NAME}/${DIST_VERSION}/$PKG_NAME" -C "${{ github.workspace }}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/" "${DIST_NAME}"
-              # calculate new hashes
-              NEW_CID=$(ipfs add -Qn "$PKG_PATH")
-              NEW_SHA512_LINE=$(gsha512sum "$PKG_PATH")
-              NEW_SHA512=$(echo "$NEW_SHA512_LINE" | gawk '{ print $1; }')
-              echo "-> New $PKG_NAME"
-              echo "   new CID:    $NEW_CID"
-              echo "   new SHA512: $NEW_SHA512"
-              # update metadata to use new hashes
-              echo "$NEW_CID" > "${PKG_PATH}.cid"
-              echo "$NEW_SHA512_LINE" > "${PKG_PATH}.sha512"
-              gsed -i "s/${OLD_CID}/${NEW_CID}/g; s/${OLD_SHA512}/${NEW_SHA512}/g" "${PKG_ROOT}/dist.json"
-              echo "-> Completed the update of ${arch}.tar.gz for ${DIST_NAME} ${DIST_VERSION}"
-            done
-          done
       - name: List ./releases after
         run: ls -Rhl ./releases || echo "No ./releases"
       - name: Temporarily save notarized artifacts
diff --git a/scripts/ci/pin-to-cluster.sh b/scripts/ci/pin-to-cluster.sh
index d4f5527b..8368b0ac 100755
--- a/scripts/ci/pin-to-cluster.sh
+++ b/scripts/ci/pin-to-cluster.sh
@@ -8,7 +8,7 @@ ipfs-cluster-ctl --enc=json \
     pin add \
     --pin-name="${PIN_NAME}" \
     --no-status $PIN_ADD_EXTRA_ARGS \
-    "PIN_CID"
+    "$PIN_CID"
 echo "::endgroup::"
 
 echo "::group::waiting until pinned"
@@ -16,7 +16,7 @@ echo "::group::waiting until pinned"
     ipfs-cluster-ctl --enc=json \
         --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
         --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
-        status "PIN_CID" | tee cluster-pin-status
+        status "$PIN_CID" | tee cluster-pin-status
     if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
         echo "Got 2 pin confirmations, finishing the workflow"
         break
diff --git a/scripts/ci/sign-new-macos-releases.sh b/scripts/ci/sign-new-macos-releases.sh
new file mode 100755
index 00000000..edfa8d31
--- /dev/null
+++ b/scripts/ci/sign-new-macos-releases.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -e
+
+echo "::group::Unpack any new darwin arm64 and amd64 binaries to ./tmp"
+    # ./releases/{DIST_NAME}/{DIST_VERSION}/*_darwin-${arch}.tar.gz
+    # -> ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
+    for NEW_DIR in ./releases/*/*; do
+        (! test -d "$NEW_DIR") && continue
+        DIST_VERSION=$(basename "$NEW_DIR")
+        DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+        for arch in "amd64" "arm64"; do
+            mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned"
+            tar -zxvf "./releases/${DIST_NAME}/${DIST_VERSION}/${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz" -C "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/"
+        done
+    done
+    ls -Rhl ./tmp || echo "Nothing new in ./tmp"
+echo "::endgroup::"
+
+echo "::group::Sign and notarize the mac binaries"
+    # Find and sign executables in
+    # ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/
+    for NEW_DIR in ./releases/*/*; do
+        (! test -d "$NEW_DIR") && continue
+        DIST_VERSION=$(basename "$NEW_DIR")
+        DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+        for arch in "amd64" "arm64"; do
+            EXECUTABLES=$(jq -nc '$ARGS.positional' --args $(find "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-unsigned/" -perm +111 -type f -print))
+            echo "{
+                \"source\" : $EXECUTABLES,
+                \"bundle_id\" : \"io.ipfs.dist.${DIST_NAME}\",
+                \"apple_id\": {
+                \"password\":  \"@env:AC_PASSWORD\"
+                },
+                \"sign\" :{
+                \"application_identity\" : \"Developer ID Application: Protocol Labs, Inc. (7Y229E2YRL)\"
+                },
+                \"zip\" :{
+                    \"output_path\" : \"./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip\"
+                }
+            }" | tee | jq > "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
+            gon -log-level=info -log-json "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-gon.json"
+        done
+    done
+echo "::endgroup::"
+
+
+echo "::group::Update changed binaries in ./releases"
+    # ./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip
+    # -> ./releases/{DIST_NAME}/{DIST_VERSION}/*_darwin-${arch}.tar.gz
+    for NEW_DIR in ./releases/*/*; do
+        (! test -d "$NEW_DIR") && continue
+        DIST_VERSION=$(basename "$NEW_DIR")
+        DIST_NAME=$(basename $(dirname "$NEW_DIR"))
+        for arch in "amd64" "arm64"; do
+            echo "-> Starting the update of darwin_${arch}.tar.gz for name='${DIST_NAME}' and version='${DIST_VERSION}'"
+            # unzip signed binaries to a directory matching .tar.gz structure
+            cd "${WORK_DIR}"
+            mkdir -p "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}"
+            cd "./tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/${DIST_NAME}/"
+            echo "-> Unpacking gon .zip for ${arch}"
+            unzip "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed.zip"
+            echo "-> Unpacked contents"
+            ls -Rhl "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/"
+            # replace .tar.gz with one that has the same structure, but signed binaries
+            PKG_NAME="${DIST_NAME}_${DIST_VERSION}_darwin-${arch}.tar.gz"
+            PKG_ROOT="${WORK_DIR}/releases/${DIST_NAME}/${DIST_VERSION}"
+            PKG_PATH="${PKG_ROOT}/${PKG_NAME}"
+            DIST_JSON="${PKG_ROOT}/dist.json"
+            # read old hashes
+            OLD_CID=$(cat "${PKG_PATH}.cid")
+            OLD_SHA512=$(gawk '{ print $1; }' < "${PKG_PATH}.sha512")
+            echo "-> Found old $PKG_NAME"
+            echo "   old CID:    $OLD_CID"
+            echo "   old SHA512: $OLD_SHA512"
+            echo "-> Updating $PKG_NAME"
+            rm "$PKG_PATH"
+            tar -czvf "${WORK_DIR}/releases/${DIST_NAME}/${DIST_VERSION}/$PKG_NAME" -C "${WORK_DIR}/tmp/${DIST_NAME}_${DIST_VERSION}_${arch}-signed/" "${DIST_NAME}"
+            # calculate new hashes
+            NEW_CID=$(ipfs add -Qn "$PKG_PATH")
+            NEW_SHA512_LINE=$(gsha512sum "$PKG_PATH")
+            NEW_SHA512=$(echo "$NEW_SHA512_LINE" | gawk '{ print $1; }')
+            echo "-> New $PKG_NAME"
+        echo "   new CID:    $NEW_CID"
+        echo "   new SHA512: $NEW_SHA512"
+        # update metadata to use new hashes
+        echo "$NEW_CID" > "${PKG_PATH}.cid"
+        echo "$NEW_SHA512_LINE" > "${PKG_PATH}.sha512"
+        gsed -i "s/${OLD_CID}/${NEW_CID}/g; s/${OLD_SHA512}/${NEW_SHA512}/g" "${PKG_ROOT}/dist.json"
+        echo "-> Completed the update of ${arch}.tar.gz for ${DIST_NAME} ${DIST_VERSION}"
+        done
+    done
+echo "::endgroup::"

From 835df84f9c608de475852fcfdbd3572b8d54b306 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 16:26:01 +0200
Subject: [PATCH 27/34] refactor: github-preview-link.sh

---
 .github/workflows/main.yml        | 17 +++++------------
 scripts/ci/github-preview-link.sh | 14 ++++++++++++++
 2 files changed, 19 insertions(+), 12 deletions(-)
 create mode 100755 scripts/ci/github-preview-link.sh

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index b0a0ed6e..84ef550b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -124,15 +124,8 @@ jobs:
           CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
         timeout-minutes: 60
       - name: Update PR status with preview link
-        run: |
-          PREVIEW_URL="https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}"
-          API_PARAMS=$(jq --monochrome-output --null-input \
-            --arg state "success" \
-            --arg target_url "$PREVIEW_URL" \
-            --arg description "Preview on IPFS" \
-            --arg context "IPFS" \
-            '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
-          curl --output /dev/null --silent --show-error \
-            -X POST -H 'Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -H 'Content-Type: application/json' \
-            --data "$API_PARAMS" 'https://api.github.com/repos/ipfs/distributions/statuses/${{ github.sha }}'
-          echo "Pinned to IPFS - $PREVIEW_URL"
+        run: ./scripts/ci/github-preview-link.sh
+        env:
+          CONTENT_PATH: "/ipfs/${{ steps.cid-reader.outputs.CID }}/"
+          GIT_REVISION: ${{ github.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/scripts/ci/github-preview-link.sh b/scripts/ci/github-preview-link.sh
new file mode 100755
index 00000000..41d92654
--- /dev/null
+++ b/scripts/ci/github-preview-link.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -e
+
+PREVIEW_URL="https://dweb.link$CONTENT_PATH"
+API_PARAMS=$(jq --monochrome-output --null-input \
+    --arg state "success" \
+    --arg target_url "$PREVIEW_URL" \
+    --arg description "Preview updated website on IPFS" \
+    --arg context "Preview is ready" \
+    '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
+curl --output /dev/null --silent --show-error \
+    -X POST -H "Authorization: Bearer $GITHUB_TOKEN" -H 'Content-Type: application/json' \
+    --data "$API_PARAMS" 'https://api.github.com/repos/ipfs/distributions/statuses/${GIT_REVISION}'
+echo "Pinned to IPFS - $PREVIEW_URL"

From 1c9b839bab7b2ec9e2422f624f875a760352486a Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 17:35:41 +0200
Subject: [PATCH 28/34] test: signed release built for ipfs-update@v1.7.1

---
 .github/workflows/main.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 84ef550b..ea0b9bc7 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -10,7 +10,8 @@ on:
 #      - master
 
 env:
- DIST_ROOT: '/ipns/dist.ipfs.io' # content root used for calculating diff to build
+ # TODO: restore DIST_ROOT: '/ipns/dist.ipfs.io' # content root used for calculating diff to build
+ DIST_ROOT: '/ipfs/QmX6J3hVtMF9Y73CKcyLHfgviEfXGFFQABWPXsD17EsBhg' # dist.ipfs.io without ipfs-update@v1.7.1
  GO_IPFS_VER: 'v0.9.1'           # go-ipfs daemon used for chunking and applying diff
  CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 

From e2570b88e625374b3e89d5ed6465eac1075085e9 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 18:41:30 +0200
Subject: [PATCH 29/34] refactor: reuse ./scripts/ci

this makes nightly do exactly what release build does, but allows us
to tweak some behaviors, for example do pins that expires after 7 days,
or report each preview separately
---
 .github/workflows/nightly.yml     | 58 ++++++++++++++-----------------
 scripts/ci/github-preview-link.sh |  6 ++--
 2 files changed, 30 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 6f000dbb..74609a07 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -12,6 +12,10 @@ env:
  GO_IPFS_VER: 'v0.9.1'           # go-ipfs daemon used for chunking and applying diff
  CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 
+concurrency:
+  group: nightly
+  cancel-in-progress: true
+
 jobs:
   prepare-matrix:
     runs-on: ubuntu-latest
@@ -26,42 +30,18 @@ jobs:
     runs-on: "ubuntu-latest"
     needs: prepare-matrix
     strategy:
+      max-parallel: 4 # avoid using all workers from the pool
       fail-fast: false
       matrix:
         dist_name: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
     steps:
       - uses: actions/checkout@v2
-      - name: Install ipfs and deps
-        run: |
-          curl -s https://dist.ipfs.io/go-ipfs/${GO_IPFS_VER}/go-ipfs_${GO_IPFS_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ go-ipfs/ipfs --strip-components=1
-          curl -s https://dist.ipfs.io/ipfs-cluster-ctl/${CLUSTER_CTL_VER}/ipfs-cluster-ctl_${CLUSTER_CTL_VER}_linux-amd64.tar.gz | sudo tar vzx -C /usr/local/bin/ ipfs-cluster-ctl/ipfs-cluster-ctl --strip-components=1
-      - name: Fix resolv # DNS provided by Github is unreliable for DNSLik/dnsaddr
-        run: sudo sed -i -e 's/nameserver 127.0.0.*/nameserver 1.1.1.1/g' /etc/resolv.conf
-      - name: Set up ipfs
-        run: |
-          ipfs init --profile flatfs,server,test,lowpower
-          # make flatfs async for faster ci
-          new_config=$( jq '.Datastore.Spec.mounts[0].child.sync = false' ~/.ipfs/config) && echo "${new_config}" > ~/.ipfs/config
-          # restore deterministic port (changed by test profile)
-          ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5001"
-      - name: Wait for ipfs daemon
-        run: ipfs daemon --enable-gc=false & while (! ipfs id --api "/ip4/127.0.0.1/tcp/5001"); do sleep 1; done
-        timeout-minutes: 3
-      - name: Preconnect to cluster peers
-        run: |
-          echo '-> preconnect to cluster peers'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            peers ls > cluster-peers-ls
-          for maddr in $(jq -r '.[].ipfs.addresses[]?' cluster-peers-ls); do
-            ipfs swarm connect "$maddr" || continue
-          done
-          echo '-> manual connect to cluster.ipfs.io'
-          ipfs swarm connect /dnsaddr/cluster.ipfs.io
-          echo '-> list swarm peers'
-          ipfs swarm peers
-        timeout-minutes: 3
+      - name: Setup IPFS
+        run: ./scripts/ci/setup-ipfs.sh
+        env:
+          CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
+          CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
+        timeout-minutes: 5
       - run: cd ./dists/${{ matrix.dist_name }} && make nightly
       - run: ./dockerized make publish
       - name: Inspect git status and contents of ./releases
@@ -94,3 +74,19 @@ jobs:
             fi
           done
         timeout-minutes: 60
+      - name: Pin new website to ipfs-websites.collab.ipfscluster.io
+        run: ./scripts/ci/pin-to-cluster.sh
+        env:
+          PIN_CID: ${{ steps.cid-reader.outputs.CID }}
+          PIN_NAME: "ipfs/distributions/nightly/${{ matrix.dist_name }}"
+          PIN_ADD_EXTRA_ARGS: "--expire-in 168h"
+          CLUSTER_USER: ${{ secrets.CLUSTER_USER }}
+          CLUSTER_PASSWORD: ${{ secrets.CLUSTER_PASSWORD }}
+        timeout-minutes: 60
+      - name: Update PR status with preview link
+        run: ./scripts/ci/github-preview-link.sh
+        env:
+          GITHUB_TITLE: "Preview for ${{ matrix.dist_name }}"
+          CONTENT_PATH: "/ipfs/${{ steps.cid-reader.outputs.CID }}/${{ matrix.dist_name }}"
+          GIT_REVISION: ${{ github.sha }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/scripts/ci/github-preview-link.sh b/scripts/ci/github-preview-link.sh
index 41d92654..18d2c348 100755
--- a/scripts/ci/github-preview-link.sh
+++ b/scripts/ci/github-preview-link.sh
@@ -1,12 +1,12 @@
 #!/usr/bin/env bash
 set -e
 
-PREVIEW_URL="https://dweb.link$CONTENT_PATH"
+PREVIEW_URL=${PREVIEW_URL:-"https://dweb.link$CONTENT_PATH"}
 API_PARAMS=$(jq --monochrome-output --null-input \
     --arg state "success" \
     --arg target_url "$PREVIEW_URL" \
-    --arg description "Preview updated website on IPFS" \
-    --arg context "Preview is ready" \
+    --arg description "${GITHUB_DESCRIPTION:-"Preview updated website on IPFS"}" \
+    --arg context "${GITHUB_TITLE:-"Preview is ready"}" \
     '{ state: $state, target_url: $target_url, description: $description, context: $context }' )
 curl --output /dev/null --silent --show-error \
     -X POST -H "Authorization: Bearer $GITHUB_TOKEN" -H 'Content-Type: application/json' \

From 947f37d6d47390c12d1b286831dcea4640b96bf7 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 20:00:59 +0200
Subject: [PATCH 30/34] fix: nightly cleanup

This runs cleanup only when we have anything to clean up
---
 build-go.sh | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/build-go.sh b/build-go.sh
index d936fbc8..2c5e1412 100755
--- a/build-go.sh
+++ b/build-go.sh
@@ -436,11 +436,13 @@ function startGoBuilds() {
 		echo ""
 	done <<< "$newVersions"
 
-	# Keep all tagged versions from repo
-	grep -v ^nightly "$versions" > "$outputDir/versions"
-
-	# Keep at most 7 nightly versions
-	grep -h ^nightly "$versions" "$existingVersions" | sort -ur | head -n7 >> "$outputDir/versions"
+	# Additional cleanup/normalization for nightly builds
+	if grep -h ^nightly "$versions" "$existingVersions" > /dev/null; then
+		# Keep all tagged versions from repo
+		grep -v ^nightly "$versions" > "$outputDir/versions"
+		# Keep at most 7 nightly versions
+		grep -h ^nightly "$versions" "$existingVersions" | sort -ur | head -n7 >> "$outputDir/versions"
+	fi
 
 	notice "build complete!"
 }

From 5277083c3690ce612e360d1ae4db03809dbc19b9 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 20:05:15 +0200
Subject: [PATCH 31/34] refactor: deduplicate pinning

---
 .github/workflows/nightly.yml | 25 -------------------------
 scripts/ci/pin-to-cluster.sh  |  2 +-
 2 files changed, 1 insertion(+), 26 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 74609a07..82cc52a9 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -49,31 +49,6 @@ jobs:
       - name: Read CID of updated DAG
         id: cid-reader
         run: echo "::set-output name=CID::$(tail -1 ./versions)"
-      - name: Pin new website to ipfs-websites.collab.ipfscluster.io for 7 days
-        run: |
-          echo 'pin to cluster'
-          ipfs-cluster-ctl --enc=json \
-            --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-            --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-            pin add \
-            --pin-name="https://github.com/ipfs/distributions/commits/${{ github.sha }}" \
-            --no-status --expire-in 168h \
-            "${{ steps.cid-reader.outputs.CID }}"
-          while true; do
-            ipfs-cluster-ctl --enc=json \
-              --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
-              --basic-auth '${{ secrets.CLUSTER_USER }}:${{ secrets.CLUSTER_PASSWORD }}' \
-              status "${{ steps.cid-reader.outputs.CID }}" | tee cluster-pin-status
-            if [[ $(jq '.peer_map[].status' cluster-pin-status | grep '"pinned"' | wc -l) -ge 2 ]]; then
-              echo "Got 2 pin confirmations, finishing the workflow"
-              echo "Pinned ${{ matrix.dist_name }} nightly to IPFS - https://dweb.link/ipfs/${{ steps.cid-reader.outputs.CID }}/${{ matrix.dist_name }}/"
-              break
-            else
-              echo "(sleeping for 15 seconds)"
-              sleep 15
-            fi
-          done
-        timeout-minutes: 60
       - name: Pin new website to ipfs-websites.collab.ipfscluster.io
         run: ./scripts/ci/pin-to-cluster.sh
         env:
diff --git a/scripts/ci/pin-to-cluster.sh b/scripts/ci/pin-to-cluster.sh
index 8368b0ac..46089ce9 100755
--- a/scripts/ci/pin-to-cluster.sh
+++ b/scripts/ci/pin-to-cluster.sh
@@ -6,7 +6,7 @@ ipfs-cluster-ctl --enc=json \
     --host "/dnsaddr/ipfs-websites.collab.ipfscluster.io" \
     --basic-auth "${CLUSTER_USER}:${CLUSTER_PASSWORD}" \
     pin add \
-    --pin-name="${PIN_NAME}" \
+    --name "${PIN_NAME}" \
     --no-status $PIN_ADD_EXTRA_ARGS \
     "$PIN_CID"
 echo "::endgroup::"

From 843feda6acbe2d1f63f4a096f11011665949e3e4 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 28 Jul 2021 23:03:24 +0200
Subject: [PATCH 32/34] feat: fill issue for every broken nightly build

---
 .github/workflows/nightly.yml | 39 ++++++++++++++++++++++++++++++++++-
 1 file changed, 38 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 82cc52a9..ccfc3387 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: "ubuntu-latest"
     needs: prepare-matrix
     strategy:
-      max-parallel: 4 # avoid using all workers from the pool
+      max-parallel: 15 # avoid using all workers from the org pool
       fail-fast: false
       matrix:
         dist_name: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}
@@ -44,6 +44,43 @@ jobs:
         timeout-minutes: 5
       - run: cd ./dists/${{ matrix.dist_name }} && make nightly
       - run: ./dockerized make publish
+      - name: Create issue if build failed
+        uses: actions/github-script@v4
+        if: ${{ failure() }}
+        with:
+          script: |
+            const title = 'Nightly build failed for ${{ matrix.dist_name }}'
+            const body = '${{ matrix.dist_name }} failed to build from the latest commit: https://github.com/ipfs/distributions/actions/runs/${{ github.run_id }}'
+            const opts = { owner: context.repo.owner, repo: context.repo.repo }
+            const response = await github.search.issuesAndPullRequests({
+              q: `repo:ipfs/distributions is:issue is:open in:title ${title}`
+            })
+            console.log('github.issuesAndPullRequests', response)
+            let link
+            if (response.data.items.length === 0) {
+              const created = await github.issues.create({ ...opts, title, body,
+                labels: ['kind/bug', 'need/triage']
+              })
+              console.log('no open issues, created a new one', created)
+              link = created.html_url
+            }
+            for (const issue of response.data.items) {
+              if (issue.title !== title) continue
+              console.log('found existing open issue', issue)
+              const created = await github.issues.createComment({ ...opts,
+                issue_number: issue.number,
+                body
+              })
+              console.log('commented on existing open issue', created)
+              link = created.html_url
+            }
+            await github.repos.createCommitStatus({ ...opts,
+              sha: '${{ github.sha }}',
+              state: 'error',
+              target_url: link,
+              context: 'Problem with ${{ matrix.dist_name }}',
+              description: 'See details in the linked issue'
+            })
       - name: Inspect git status and contents of ./releases
         run: git status && ls -Rhl ./releases
       - name: Read CID of updated DAG

From 08caa11b460030d0c3aa3a50277b050526bbe044 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Thu, 29 Jul 2021 14:20:43 +0200
Subject: [PATCH 33/34] fix: github status link

---
 .github/workflows/nightly.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index ccfc3387..138b300a 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -62,7 +62,7 @@ jobs:
                 labels: ['kind/bug', 'need/triage']
               })
               console.log('no open issues, created a new one', created)
-              link = created.html_url
+              link = created.data.html_url
             }
             for (const issue of response.data.items) {
               if (issue.title !== title) continue
@@ -72,7 +72,7 @@ jobs:
                 body
               })
               console.log('commented on existing open issue', created)
-              link = created.html_url
+              link = created.data.html_url
             }
             await github.repos.createCommitStatus({ ...opts,
               sha: '${{ github.sha }}',

From 6a29155a3c1039fe6e35e8121dd62b0361848f46 Mon Sep 17 00:00:00 2001
From: Marcin Rataj <lidel@lidel.org>
Date: Wed, 4 Aug 2021 01:14:02 +0200
Subject: [PATCH 34/34] chore: restore target trigger (cron and manual)

---
 .github/workflows/nightly.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 138b300a..9b5039dd 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -1,14 +1,17 @@
 name: Nightly
 
 on:
-  push:
-  ## Allow manual invocation
-  #workflow_dispatch:
-  #schedule:
-  #  - cron: '0 5 * * *' # UTC
+  workflow_dispatch:
+    inputs:
+      dist_root:
+        description: 'DIST_ROOT'
+        required: true
+        default: '/ipns/dist.ipfs.io'
+  schedule:
+    - cron: '0 5 * * *' # UTC
 
 env:
- DIST_ROOT: '/ipns/dist.ipfs.io' # content root used for calculating diff to build
+ DIST_ROOT: ${{ github.event.inputs.custom_dist_root || '/ipns/dist.ipfs.io' }} # content root used for calculating diff to build
  GO_IPFS_VER: 'v0.9.1'           # go-ipfs daemon used for chunking and applying diff
  CLUSTER_CTL_VER: 'v0.14.0'      # ipfs-cluster-ctl used for pinning
 
@@ -30,7 +33,6 @@ jobs:
     runs-on: "ubuntu-latest"
     needs: prepare-matrix
     strategy:
-      max-parallel: 15 # avoid using all workers from the org pool
       fail-fast: false
       matrix:
         dist_name: ${{ fromJson(needs.prepare-matrix.outputs.matrix) }}