diff --git a/.circleci/config.yml b/.circleci/config.yml index 4057049dac..13d4ea996f 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -715,7 +715,7 @@ commands: command: | if [ "${CIRCLE_BRANCH}" = "rel/nightly" ] then - export NO_BUILD="true" + export NIGHTLY_BUILD="true" fi export PATH=$(echo "$PATH" | sed -e "s|:${HOME}/\.go_workspace/bin||g" | sed -e 's|:/usr/local/go/bin||g') export GOPATH="<< parameters.build_dir >>/go" diff --git a/Makefile b/Makefile index 5204f24345..52c505b301 100644 --- a/Makefile +++ b/Makefile @@ -179,11 +179,16 @@ ifeq ($(OS_TYPE),darwin) mkdir -p $(GOPATH1)/bin-darwin-arm64 CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE) + # same for buildsrc-special + cd tools/block-generator && \ + CROSS_COMPILE_ARCH=amd64 GOBIN=$(GOPATH1)/bin-darwin-amd64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch x86_64 -mmacos-version-min=12.0" --host=x86_64-apple-darwin' $(MAKE) + CROSS_COMPILE_ARCH=arm64 GOBIN=$(GOPATH1)/bin-darwin-arm64 MACOSX_DEPLOYMENT_TARGET=12.0 EXTRA_CONFIGURE_FLAGS='CFLAGS="-arch arm64 -mmacos-version-min=12.0" --host=aarch64-apple-darwin' $(MAKE) + # lipo together - mkdir -p $(GOPATH1)/bin-darwin-universal + mkdir -p $(GOPATH1)/bin for binary in $$(ls $(GOPATH1)/bin-darwin-arm64); do \ if [ -f $(GOPATH1)/bin-darwin-amd64/$$binary ]; then \ - lipo -create -output $(GOPATH1)/bin-darwin-universal/$$binary \ + lipo -create -output $(GOPATH1)/bin/$$binary \ $(GOPATH1)/bin-darwin-arm64/$$binary \ $(GOPATH1)/bin-darwin-amd64/$$binary; \ else \ @@ -191,7 +196,7 @@ ifeq ($(OS_TYPE),darwin) fi \ done else - $(error OS_TYPE must be darwin for universal builds) + echo "OS_TYPE must be darwin for universal builds, skipping" endif deps: diff --git a/cmd/updater/update.sh b/cmd/updater/update.sh index 6c20d55c2d..c7f049eec9 100755 --- a/cmd/updater/update.sh +++ b/cmd/updater/update.sh @@ -185,10 +185,6 @@ function get_updater_url() { UNAME=$(uname -m) if [[ "${UNAME}" = "x86_64" ]]; then ARCH="amd64" - elif [[ "${UNAME}" = "armv6l" ]]; then - ARCH="arm" - elif [[ "${UNAME}" = "armv7l" ]]; then - ARCH="arm" elif [[ "${UNAME}" = "aarch64" ]]; then ARCH="arm64" else diff --git a/package-upload.yaml b/package-upload.yaml deleted file mode 100644 index 9d13c458ef..0000000000 --- a/package-upload.yaml +++ /dev/null @@ -1,23 +0,0 @@ -tasks: - - task: s3.BucketCopy - name: amd64 - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/amd64 - dest: s3://$STAGING/$CHANNEL/$VERSION/ - - - task: s3.BucketCopy - name: arm - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/arm - dest: s3://$STAGING/$CHANNEL/$VERSION/ - - - task: s3.BucketCopy - name: arm64 - src: $HOME/projects/go-algorand/tmp/node_pkgs/linux/arm64 - dest: s3://$STAGING/$CHANNEL/$VERSION/ - -jobs: - package-upload: - tasks: - - s3.BucketCopy.amd64 - - s3.BucketCopy.arm - - s3.BucketCopy.arm64 - diff --git a/scripts/release/mule/Makefile.mule b/scripts/release/mule/Makefile.mule index 0c73cccf89..06fe5d09b7 100644 --- a/scripts/release/mule/Makefile.mule +++ b/scripts/release/mule/Makefile.mule @@ -1,15 +1,13 @@ # This file is imported into go-algorand/Makefile. PKG_DIR = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH) +PKG_DIR_UNIVERSAL = $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/universal -.PHONY: ci-clean ci-setup ci-build +.PHONY: ci-clean ci-build ci-clean: clean rm -rf tmp -ci-setup: - mkdir -p $(PKG_DIR) - ci-test: ifeq ($(ARCH), amd64) RACE=-race @@ -28,7 +26,16 @@ ci-integration: SRCROOT=$(SRCPATH) \ test/scripts/e2e.sh -c $(CHANNEL) -n -ci-build: ci-clean build ci-setup +ci-build-universal: ci-clean universal + echo $(PKG_DIR_UNIVERSAL) + mkdir -p $(PKG_DIR_UNIVERSAL) + CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR_UNIVERSAL) NO_BUILD=True VARIATIONS=$(OS_TYPE)-universal \ + scripts/build_packages.sh $(OS_TYPE)/universal && \ + mkdir -p $(PKG_DIR_UNIVERSAL)/data && \ + cp installer/genesis/devnet/genesis.json $(PKG_DIR_UNIVERSAL)/data + +ci-build: ci-clean + mkdir -p $(PKG_DIR) CHANNEL=$(CHANNEL) PKG_ROOT=$(PKG_DIR) NO_BUILD=True VARIATIONS=$(OS_TYPE)-$(ARCH) \ scripts/build_packages.sh $(OS_TYPE)/$(ARCH) && \ mkdir -p $(PKG_DIR)/data && \ diff --git a/scripts/release/mule/README.md b/scripts/release/mule/README.md index 8439767725..030728ef70 100644 --- a/scripts/release/mule/README.md +++ b/scripts/release/mule/README.md @@ -25,7 +25,6 @@ In addition, make sure that the following AWS credentials are set in environment # Build Stages - [package](#package) -- [upload](#upload) - [test](#test) - [sign](#sign) - [deploy](#deploy) @@ -49,20 +48,6 @@ In addition, make sure that the following AWS credentials are set in environment - package-docker + packages docker image -## upload - -- see `./go-algorand/package-upload.yaml` - -- customizable environment variables: - - + `CHANNEL` - + `STAGING` - + `VERSION` - -#### `mule` jobs - - - package-upload - ## test - see `./go-algorand/package-test.yaml` @@ -180,10 +165,6 @@ Let's look at some examples. mule -f package.yaml package -### Uploading - - STAGING=the-staging-area CHANNEL=beta VERSION=latest mule -f package-upload.yaml package-upload - ### Testing 1. As part of the test suite, the `verify_package_string.sh` test needs the `BRANCH` as well as the `SHA`: diff --git a/scripts/travis/deploy_packages.sh b/scripts/travis/deploy_packages.sh index e7e517394c..5879835b01 100755 --- a/scripts/travis/deploy_packages.sh +++ b/scripts/travis/deploy_packages.sh @@ -11,7 +11,7 @@ set -e SCRIPTPATH="$( cd "$(dirname "$0")" ; pwd -P )" - +OSARCH=$("${SCRIPTPATH}/../osarchtype.sh") # Get the go build version. GOLANG_VERSION=$("${SCRIPTPATH}/../get_golang_version.sh") @@ -24,6 +24,15 @@ then exit 1 fi +if [ "${NIGHTLY_BUILD}" == "true" ]; then + # we want to rebuild universal binaries for nightly builds + NO_BUILD=true + if [ "${OSARCH}" == "darwin/arm64" ]; then + make universal + OSARCH="darwin/universal" + fi +fi + if [ -z "${NO_BUILD}" ] || [ "${NO_BUILD}" != "true" ]; then scripts/travis/build.sh fi @@ -31,4 +40,4 @@ fi export RELEASE_GENESIS_PROCESS=true export NO_BUILD=true export SkipCleanCheck=1 -scripts/deploy_version.sh "${TRAVIS_BRANCH}" "$(./scripts/osarchtype.sh)" +scripts/deploy_version.sh "${TRAVIS_BRANCH}" "${OSARCH}" diff --git a/test/muleCI/mule.yaml b/test/muleCI/mule.yaml index afa73ae2c1..5022cba792 100644 --- a/test/muleCI/mule.yaml +++ b/test/muleCI/mule.yaml @@ -93,11 +93,8 @@ agents: tasks: - task: shell.Make - name: build.darwin-arm64 - target: ci-build - - task: shell.Make - name: build.darwin-amd64 - target: ci-build + name: build.darwin-universal + target: ci-build-universal - task: docker.Make name: build.amd64 agent: cicd.ubuntu.amd64 @@ -124,9 +121,9 @@ tasks: # Stash tasks - task: stash.Stash - name: darwin-arm64 + name: darwin-universal bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64 + stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal globSpecs: - tmp/node_pkgs/**/* - task: stash.Stash @@ -135,12 +132,6 @@ tasks: stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64 globSpecs: - tmp/node_pkgs/**/* - - task: stash.Stash - name: darwin-amd64 - bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64 - globSpecs: - - tmp/node_pkgs/**/* - task: stash.Stash name: linux-arm64 bucketName: go-algorand-ci-cache @@ -164,13 +155,9 @@ tasks: bucketName: go-algorand-ci-cache stashId: ${JENKINS_JOB_CACHE_ID}/linux-amd64 - task: stash.Unstash - name: darwin-amd64 + name: darwin-universal bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-amd64 - - task: stash.Unstash - name: darwin-arm64 - bucketName: go-algorand-ci-cache - stashId: ${JENKINS_JOB_CACHE_ID}/darwin-arm64 + stashId: ${JENKINS_JOB_CACHE_ID}/darwin-universal - task: stash.Unstash name: packages bucketName: go-algorand-ci-cache @@ -187,14 +174,10 @@ tasks: target: mule-sign jobs: - build-darwin-arm64: - tasks: - - shell.Make.build.darwin-arm64 - - stash.Stash.darwin-arm64 - build-darwin-amd64: + build-darwin-universal: tasks: - - shell.Make.build.darwin-amd64 - - stash.Stash.darwin-amd64 + - shell.Make.build.darwin-universal + - stash.Stash.darwin-universal build-linux-amd64: tasks: - docker.Make.build.amd64 @@ -207,8 +190,7 @@ jobs: tasks: - stash.Unstash.linux-amd64 - stash.Unstash.linux-arm64 - - stash.Unstash.darwin-arm64 - - stash.Unstash.darwin-amd64 + - stash.Unstash.darwin-universal - docker.Make.deb.amd64 - docker.Make.rpm.amd64 - stash.Stash.packages diff --git a/util/s3/s3Helper.go b/util/s3/s3Helper.go index efad67d924..79396e7ced 100644 --- a/util/s3/s3Helper.go +++ b/util/s3/s3Helper.go @@ -189,7 +189,17 @@ func (helper *Helper) GetPackageVersion(channel string, pkg string, specificVers osName := runtime.GOOS arch := runtime.GOARCH prefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, arch) - return helper.GetPackageFilesVersion(channel, prefix, specificVersion) + + maxVersion, maxVersionName, err = helper.GetPackageFilesVersion(channel, prefix, specificVersion) + // For darwin, we want to also look at universal binaries + if osName == "darwin" { + universalPrefix := fmt.Sprintf("%s_%s_%s-%s_", pkg, channel, osName, "universal") + universalMaxVersion, universalMaxVersionName, universalErr := helper.GetPackageFilesVersion(channel, universalPrefix, specificVersion) + if universalMaxVersion > maxVersion { + return universalMaxVersion, universalMaxVersionName, universalErr + } + } + return maxVersion, maxVersionName, err } // GetPackageFilesVersion return the package version