From 94c119f197a83f30c9e0bfb0c1ea01bc5c88549d Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 09:06:22 +0200 Subject: [PATCH 01/20] ci: migrate ci setup from circle ci to github actions --- .../actions/configure-environment/action.yml | 40 ++++ .github/workflows/ci.yml | 182 ++++++++++++++++++ 2 files changed, 222 insertions(+) create mode 100644 .github/actions/configure-environment/action.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml new file mode 100644 index 00000000..230be0ff --- /dev/null +++ b/.github/actions/configure-environment/action.yml @@ -0,0 +1,40 @@ +name: Configure Environment Variables +description: Configure environment variables for Filecoin FFI + +runs: + using: 'composite' + steps: + - run: | + echo 'FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/"' >> $GITHUB_ENV + echo 'GO111MODULE=on' >> $GITHUB_ENV + echo 'GOPATH="${GITHUB_WORKSPACE}/go"' >> $GITHUB_ENV + echo 'PATH="/usr/local/go/bin:${GITHUB_WORKSPACE}/.cargo/bin:${PATH}:${GITHUB_WORKSPACE}/go/bin:${GITHUB_WORKSPACE}/.bin"' >> $GITHUB_ENV + echo 'RUST_LOG=info' >> $GITHUB_ENV + echo 'CIRCLE_ARTIFACTS="/tmp"' >> $GITHUB_ENV + # Make sure CUDA is found on aarch64 + echo 'LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:${LD_LIBRARY_PATH}"' >> $GITHUB_ENV + echo 'LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:${LIBRARY_PATH}"' >> $GITHUB_ENV + shell: bash + - if: runner.os == 'MacOS' + run: | + echo 'export PATH="${GITHUB_WORKSPACE}/.cargo/bin:${GITHUB_WORKSPACE}/.bin:${PATH}"' >> $GITHUB_ENV + echo 'export LIBRARY_PATH=/opt/homebrew/lib' >> $GITHUB_ENV + shell: bash + - if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y valgrind ocl-icd-opencl-dev libssl-dev libhwloc-dev nvidia-cuda-toolkit g++-10 + # Downgrade to GCC 10, as CUDA 11 doesn't play nice with GCC 11 + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 10 + sudo update-alternatives --set c++ /usr/bin/g++-10 + shell: bash + - if: runner.os == 'MacOS' + run: | + brew install filippo.io/age/age + shell: bash + - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 + with: + toolchain: 1.73 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..ca9be0e4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,182 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + tags: + - v* + workflow_dispatch: + inputs: + publish: + description: 'Publish the static library' + required: false + default: 'false' + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +# Can we apply these to the entire workflow? +env: + # Build the kernel only for the single architecture. This should reduce + # the overall compile-time significantly. + EC_GPU_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + +jobs: + check: + name: Check code style and run linters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - name: Run shellcheck + run: shellcheck ./install-filcrypto + - name: Run cargo fmt + run: cargo fmt --manifest-path ./rust/Cargo.toml --all -- --check + - name: Run cargo clippy + run: cd rust && cargo clippy --all-targets --features blst-portable,opencl -- -D warnings + - name: Run go fmt + # `! go fmt ./... 2>&1 | read"` doesn't work, this one does, thanks + # https://carsonip.me/posts/go-fmt-and-ci/ + run: | + output=$(go fmt ./...) + echo "${output}" + test -z "${output}" + - name: Run various linters + run: make go-lint + cgo-bindings: + name: Build and test CGO bindings (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['ubuntu-latest', ["self-hosted", "linux", "arm64", "xlarge"], 'macos-latest'] + # # Is it OK if we operate in the default working directory? + # defaults: + # run: + # working-directory: go/src/github.com/filecoin-project/filecoin-ffi + steps: + - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: runner.os == 'MacOS' + run: cd rust && cargo fetch + - name: Build project + run: make + - name: Build project without CGO + run: env CGO_ENABLED=0 go build . + - if: runner.os == 'Linux' + uses: actions/cache/restore@v3 + with: + key: v28-proof-params-${{ runner.os }}-${{ runner.arch }} + path: | + ./filecoin-proof-parameters + - if: runner.os == 'Linux' + name: Obtain Filecoin parameters + run: | + DIR=$(pwd) + cd $(mktemp -d) + go install github.com/filecoin-project/go-paramfetch/paramfetch@latest + $GOPATH/bin/paramfetch 2048 "${DIR}/parameters.json" "${DIR}/srs-inner-product.json" + - if: runner.os == 'Linux' && github.event == 'push' && !startsWith(github.ref, 'refs/tags/') + uses: actions/cache/save@v3 + with: + key: v28-proof-params-${{ runner.os }}-${{ runner.arch }} + path: | + ./filecoin-proof-parameters + - if: runner.os == 'Linux' + run: cd rust && rustup target add wasm32-unknown-unknown + - if: runner.os == 'Linux' && runner.arch == 'x86_64' + run: make cgo-leakdetect + - if: runner.os == 'Linux' + run: cd rust && FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/" RUST_LOG=info cargo test --all --release -- --test-threads 1 && cd .. + - if: runner.os == 'Linux' + run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -p 1 -timeout 60m + - if: runner.os == 'MacOS' + name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin) + run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -run=^$ + supraseal: + runs-on: ubuntu-latest + # # Is it OK if we operate in the default working directory? + # defaults: + # run: + # working-directory: go/src/github.com/filecoin-project/filecoin-ffi + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - name: Build project with `FFI_USE_CUDA_SUPRASEAL=1` + run: FFI_BUILD_FROM_SOURCE=1 FFI_USE_CUDA_SUPRASEAL=1 make + publish: + needs: [check, cgo-bindings, supraseal] + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || github.event.inputs.publish == 'true' + name: Publish the static library (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['ubuntu-latest', ['self-hosted', 'linux', 'arm64', 'xlarge'], 'macos-latest'] + steps: + - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: runner.os == 'MacOS' + run: | + cd rust && rustup target add x86_64-apple-darwin + cd rust && cargo fetch + - if: runner.os == 'Linux' + name: Build and publish the standard release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + TARBALL_PATH="/tmp/${REPOSITORY_NAME}-$(uname)-$(uname -m)-standard.tar.gz" + RELEASE_NAME="${REPOSITORY_NAME}-$(uname)-$(uname -m)-standard" + + # Note: the blst dependency uses the portable configuration for maximum compatibility + ./scripts/build-release.sh build --verbose --no-default-features --features multicore-sdr,opencl,blst-portable + ./scripts/package-release.sh $TARBALL_PATH + ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME + - if: runner.os == 'Linux' + name: Build the optimized release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-$(uname -m)-optimized.tar.gz" + RUSTFLAGS="-C target-feature=$(cat rustc-target-features-optimized.json | jq -r '.[].rustc_target_feature' | tr '\n' ',')" + + ./scripts/build-release.sh build --verbose --no-default-features --features multicore-sdr,opencl + ./scripts/package-release.sh $TARBALL_PATH + - if: runner.os == 'MacOS' + name: Build and publish the universal standard release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + RELEASE_NAME="${CIRCLE_PROJECT_REPONAME}-$(uname)-standard" + TARBALL_PATH="/tmp/${RELEASE_NAME}.tar.gz" + + # Note: the blst dependency uses the portable configuration for maximum compatibility + ./scripts/build-release.sh lipo --verbose --no-default-features --features multicore-sdr,opencl,blst-portable + ./scripts/package-release.sh $TARBALL_PATH + ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME From 53ba34ba7daf260e4a72e69c3f4c6fb5867888a3 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 09:20:44 +0200 Subject: [PATCH 02/20] fix: the env var setup for the ci workflow --- .github/actions/configure-environment/action.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 230be0ff..2d6734c4 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -5,19 +5,18 @@ runs: using: 'composite' steps: - run: | - echo 'FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/"' >> $GITHUB_ENV + echo "FIL_PROOFS_PARAMETER_CACHE=${GITHUB_WORKSPACE}/filecoin-proof-parameters/" >> $GITHUB_ENV echo 'GO111MODULE=on' >> $GITHUB_ENV - echo 'GOPATH="${GITHUB_WORKSPACE}/go"' >> $GITHUB_ENV - echo 'PATH="/usr/local/go/bin:${GITHUB_WORKSPACE}/.cargo/bin:${PATH}:${GITHUB_WORKSPACE}/go/bin:${GITHUB_WORKSPACE}/.bin"' >> $GITHUB_ENV echo 'RUST_LOG=info' >> $GITHUB_ENV - echo 'CIRCLE_ARTIFACTS="/tmp"' >> $GITHUB_ENV + shell: bash + - if: runner.os == 'Linux' && runner.arch == 'x86_64' + run: | # Make sure CUDA is found on aarch64 - echo 'LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:${LD_LIBRARY_PATH}"' >> $GITHUB_ENV - echo 'LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:${LIBRARY_PATH}"' >> $GITHUB_ENV + echo 'LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LD_LIBRARY_PATH}"'"' >> $GITHUB_ENV + echo 'LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LIBRARY_PATH}"'"' >> $GITHUB_ENV shell: bash - if: runner.os == 'MacOS' run: | - echo 'export PATH="${GITHUB_WORKSPACE}/.cargo/bin:${GITHUB_WORKSPACE}/.bin:${PATH}"' >> $GITHUB_ENV echo 'export LIBRARY_PATH=/opt/homebrew/lib' >> $GITHUB_ENV shell: bash - if: runner.os == 'Linux' @@ -30,7 +29,7 @@ runs: shell: bash - if: runner.os == 'MacOS' run: | - brew install filippo.io/age/age + HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc || brew link --overwrite python@2 shell: bash - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 with: From 1b268ec620c0c180d35085d30ce645ef50281c7e Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 09:39:49 +0200 Subject: [PATCH 03/20] fix: use the correct casing for os checks --- .github/actions/configure-environment/action.yml | 6 +++--- .github/workflows/ci.yml | 16 ++++++++++------ 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 2d6734c4..22daad42 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -9,13 +9,13 @@ runs: echo 'GO111MODULE=on' >> $GITHUB_ENV echo 'RUST_LOG=info' >> $GITHUB_ENV shell: bash - - if: runner.os == 'Linux' && runner.arch == 'x86_64' + - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: | # Make sure CUDA is found on aarch64 echo 'LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LD_LIBRARY_PATH}"'"' >> $GITHUB_ENV echo 'LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LIBRARY_PATH}"'"' >> $GITHUB_ENV shell: bash - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' run: | echo 'export LIBRARY_PATH=/opt/homebrew/lib' >> $GITHUB_ENV shell: bash @@ -27,7 +27,7 @@ runs: sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 10 sudo update-alternatives --set c++ /usr/bin/g++-10 shell: bash - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc || brew link --overwrite python@2 shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca9be0e4..09c681eb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,9 @@ jobs: - name: Run shellcheck run: shellcheck ./install-filcrypto - name: Run cargo fmt - run: cargo fmt --manifest-path ./rust/Cargo.toml --all -- --check + run: | + rustup component add rustfmt + cargo fmt --manifest-path ./rust/Cargo.toml --all -- --check - name: Run cargo clippy run: cd rust && cargo clippy --all-targets --features blst-portable,opencl -- -D warnings - name: Run go fmt @@ -63,6 +65,7 @@ jobs: strategy: matrix: runner: ['ubuntu-latest', ["self-hosted", "linux", "arm64", "xlarge"], 'macos-latest'] + fail-fast: false # # Is it OK if we operate in the default working directory? # defaults: # run: @@ -73,7 +76,7 @@ jobs: with: submodules: recursive - uses: ./.github/actions/configure-environment - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' run: cd rust && cargo fetch - name: Build project run: make @@ -100,13 +103,13 @@ jobs: ./filecoin-proof-parameters - if: runner.os == 'Linux' run: cd rust && rustup target add wasm32-unknown-unknown - - if: runner.os == 'Linux' && runner.arch == 'x86_64' + - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: make cgo-leakdetect - if: runner.os == 'Linux' run: cd rust && FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/" RUST_LOG=info cargo test --all --release -- --test-threads 1 && cd .. - if: runner.os == 'Linux' run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -p 1 -timeout 60m - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin) run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -run=^$ supraseal: @@ -130,13 +133,14 @@ jobs: strategy: matrix: runner: ['ubuntu-latest', ['self-hosted', 'linux', 'arm64', 'xlarge'], 'macos-latest'] + fail-fast: false steps: - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" - uses: actions/checkout@v4 with: submodules: recursive - uses: ./.github/actions/configure-environment - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' run: | cd rust && rustup target add x86_64-apple-darwin cd rust && cargo fetch @@ -166,7 +170,7 @@ jobs: ./scripts/build-release.sh build --verbose --no-default-features --features multicore-sdr,opencl ./scripts/package-release.sh $TARBALL_PATH - - if: runner.os == 'MacOS' + - if: runner.os == 'macOS' name: Build and publish the universal standard release run: | cd rust From 2c31e5b547d8d33684fe6be3baaa471fdb0b2798 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 10:02:59 +0200 Subject: [PATCH 04/20] fix: gopath setting for the ci workflow --- .github/actions/configure-environment/action.yml | 4 +++- .github/workflows/ci.yml | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 22daad42..3b75b3be 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -8,6 +8,7 @@ runs: echo "FIL_PROOFS_PARAMETER_CACHE=${GITHUB_WORKSPACE}/filecoin-proof-parameters/" >> $GITHUB_ENV echo 'GO111MODULE=on' >> $GITHUB_ENV echo 'RUST_LOG=info' >> $GITHUB_ENV + echo "GOPATH=${HOME}/go" >> $GITHUB_ENV shell: bash - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: | @@ -17,7 +18,8 @@ runs: shell: bash - if: runner.os == 'macOS' run: | - echo 'export LIBRARY_PATH=/opt/homebrew/lib' >> $GITHUB_ENV + echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV shell: bash - if: runner.os == 'Linux' run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 09c681eb..e3822584 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ on: - v* workflow_dispatch: inputs: + save: + description: 'Save Filecoin parameters' + required: false + default: 'false' publish: description: 'Publish the static library' required: false @@ -95,7 +99,7 @@ jobs: cd $(mktemp -d) go install github.com/filecoin-project/go-paramfetch/paramfetch@latest $GOPATH/bin/paramfetch 2048 "${DIR}/parameters.json" "${DIR}/srs-inner-product.json" - - if: runner.os == 'Linux' && github.event == 'push' && !startsWith(github.ref, 'refs/tags/') + - if: runner.os == 'Linux' && ((github.event == 'push' && !startsWith(github.ref, 'refs/tags/')) || github.event.inputs.save == 'true') uses: actions/cache/save@v3 with: key: v28-proof-params-${{ runner.os }}-${{ runner.arch }} @@ -113,6 +117,7 @@ jobs: name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin) run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -run=^$ supraseal: + name: Build with CUDA supraseal runs-on: ubuntu-latest # # Is it OK if we operate in the default working directory? # defaults: From 98715d9da2433a21733ea5ee1568c3befc949289 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 10:22:02 +0200 Subject: [PATCH 05/20] fix: put go and rust bin on path --- .github/actions/configure-environment/action.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 3b75b3be..20009799 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -10,6 +10,12 @@ runs: echo 'RUST_LOG=info' >> $GITHUB_ENV echo "GOPATH=${HOME}/go" >> $GITHUB_ENV shell: bash + - run: | + echo "/usr/local/go/bin" >> $GITHUB_PATH + echo "${GOPATH}/bin" >> $GITHUB_PATH + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + echo "${HOME}/.bin" >> $GITHUB_PATH + shell: bash - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: | # Make sure CUDA is found on aarch64 From ce16d250d44717153b76d535e5ff73a67b60fcc7 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 11:10:37 +0200 Subject: [PATCH 06/20] fix: install cuda on linux arm, change xcode on macos and install golangci-lint --- .github/actions/configure-environment/action.yml | 8 +++----- .github/workflows/ci.yml | 4 +++- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 20009799..7348c2a7 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -17,13 +17,11 @@ runs: echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash - if: runner.os == 'Linux' && runner.arch == 'ARM64' - run: | - # Make sure CUDA is found on aarch64 - echo 'LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LD_LIBRARY_PATH}"'"' >> $GITHUB_ENV - echo 'LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/stubs:'"${LIBRARY_PATH}"'"' >> $GITHUB_ENV - shell: bash + uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16 - if: runner.os == 'macOS' run: | + ls "/Applications" | grep "Xcode_" + sudo xcode-select -s "/Applications/Xcode_14.3.1.app" echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV echo "LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e3822584..9ee9f2e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,7 +62,9 @@ jobs: echo "${output}" test -z "${output}" - name: Run various linters - run: make go-lint + run: | + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.56.2 + make go-lint cgo-bindings: name: Build and test CGO bindings (${{ matrix.runner }}) runs-on: ${{ matrix.runner }} From dedd25a1cc2fa39b4f576016b64ff36ca6934946 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 11:30:29 +0200 Subject: [PATCH 07/20] fix: install libmx --- .github/actions/configure-environment/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 7348c2a7..09a89aed 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -35,7 +35,7 @@ runs: shell: bash - if: runner.os == 'macOS' run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc || brew link --overwrite python@2 + HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc libmx || brew link --overwrite python@2 shell: bash - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 with: From 2129b06272913d564504310588d2fa40d812f9e5 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 11:44:25 +0200 Subject: [PATCH 08/20] fix: install cuda on linux arm --- .github/actions/configure-environment/action.yml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 09a89aed..9ed4499d 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -16,8 +16,6 @@ runs: echo "${HOME}/.cargo/bin" >> $GITHUB_PATH echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash - - if: runner.os == 'Linux' && runner.arch == 'ARM64' - uses: Jimver/cuda-toolkit@9b295696791d75d658d8de64c4a85097ad8abeaf # v0.2.16 - if: runner.os == 'macOS' run: | ls "/Applications" | grep "Xcode_" @@ -37,6 +35,17 @@ runs: run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc libmx || brew link --overwrite python@2 shell: bash + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin + sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 + wget https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo dpkg -i cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ + sudo apt-get update + sudo apt-get -y install cuda-toolkit-12-5 + echo "LD_LIBRARY_PATH=/usr/local/cuda-12.5/lib64:/usr/local/cuda-12.5/lib64/stubs:$LD_LIBRARY_PATH" >> $GITHUB_ENV + shell: bash - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 with: toolchain: 1.73 From b8c6e28e6218a07d051735b17ef2230d384f1bbc Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 11:49:04 +0200 Subject: [PATCH 09/20] fix: remove unused variable from bls_test --- bls_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bls_test.go b/bls_test.go index c013803a..13634258 100644 --- a/bls_test.go +++ b/bls_test.go @@ -121,12 +121,10 @@ func BenchmarkBLSVerifyBatch(b *testing.B) { func benchmarkBLSVerifyBatchSize(size int) func(b *testing.B) { return func(b *testing.B) { var digests []Digest - var msgs []Message var sigs []Signature var pubks []PublicKey for i := 0; i < size; i++ { msg := Message(fmt.Sprintf("cats cats cats cats %d %d %d dogs", i, i, i)) - msgs = append(msgs, msg) digests = append(digests, Hash(msg)) priv := PrivateKeyGenerate() sig := PrivateKeySign(priv, msg) From c167a6bb33060664531ffd3c40abaa8426c7769c Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 12:10:23 +0200 Subject: [PATCH 10/20] fix: link libcuda.so correctly --- .github/actions/configure-environment/action.yml | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 9ed4499d..d0b046bc 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -20,8 +20,9 @@ runs: run: | ls "/Applications" | grep "Xcode_" sudo xcode-select -s "/Applications/Xcode_14.3.1.app" - echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV + + echo "CPATH=$(brew --prefix)/include:$CPATH" | tee -a $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix)/lib:$LIBRARY_PATH" | tee -a $GITHUB_ENV shell: bash - if: runner.os == 'Linux' run: | @@ -33,18 +34,21 @@ runs: shell: bash - if: runner.os == 'macOS' run: | - HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc libmx || brew link --overwrite python@2 + HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc shell: bash - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: | - wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin + wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 - wget https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + wget -q https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb sudo dpkg -i cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ sudo apt-get update sudo apt-get -y install cuda-toolkit-12-5 echo "LD_LIBRARY_PATH=/usr/local/cuda-12.5/lib64:/usr/local/cuda-12.5/lib64/stubs:$LD_LIBRARY_PATH" >> $GITHUB_ENV + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/stubs/libcuda.so.1 + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/libcuda.so.1 + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/libcuda.so shell: bash - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 with: From 5df5d6fdfe1ef6bd2a8c96000a42394aaf70c4a3 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 13:29:46 +0200 Subject: [PATCH 11/20] fix: install pkg-config on linux --- .../actions/configure-environment/action.yml | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index d0b046bc..11e5b4b2 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -9,6 +9,9 @@ runs: echo 'GO111MODULE=on' >> $GITHUB_ENV echo 'RUST_LOG=info' >> $GITHUB_ENV echo "GOPATH=${HOME}/go" >> $GITHUB_ENV + echo "CPATH=/usr/include:/usr/local/include:$CPATH" >> $GITHUB_ENV + echo "LIBRARY_PATH=/usr/lib:/usr/local/lib:$LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV shell: bash - run: | echo "/usr/local/go/bin" >> $GITHUB_PATH @@ -16,21 +19,37 @@ runs: echo "${HOME}/.cargo/bin" >> $GITHUB_PATH echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash + - if: runner.os == 'Linux' && runner.arch == 'X64' + run: | + echo "CPATH=/usr/include/x86_64-linux-gnu:$CPATH" >> $GITHUB_ENV + echo "LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH" >> $GITHUB_ENV + shell: bash + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + echo "CPATH=/usr/include/aarch64-linux-gnu:$CPATH" >> $GITHUB_ENV + echo "LIBRARY_PATH=/usr/lib/aarch64-linux-gnu:$LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH" >> $GITHUB_ENV + shell: bash - if: runner.os == 'macOS' run: | - ls "/Applications" | grep "Xcode_" - sudo xcode-select -s "/Applications/Xcode_14.3.1.app" - - echo "CPATH=$(brew --prefix)/include:$CPATH" | tee -a $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix)/lib:$LIBRARY_PATH" | tee -a $GITHUB_ENV + echo "CPATH=$(brew --prefix)/include:$CPATH" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix)/lib:$LIBRARY_PATH" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$(brew --prefix)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV shell: bash - if: runner.os == 'Linux' run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y valgrind ocl-icd-opencl-dev libssl-dev libhwloc-dev nvidia-cuda-toolkit g++-10 + sudo apt-get install --no-install-recommends -y valgrind ocl-icd-opencl-dev libssl-dev libhwloc-dev nvidia-cuda-toolkit g++-10 pkgconf # Downgrade to GCC 10, as CUDA 11 doesn't play nice with GCC 11 sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 10 sudo update-alternatives --set c++ /usr/bin/g++-10 + # Check if we need to install cuda-toolkit-12-5 + shell: bash + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + ls /usr/include/aarch64-linux-gnu + ls /usr/lib/aarch64-linux-gnu shell: bash - if: runner.os == 'macOS' run: | From 540c30248d878b8dc3a4d4fb6c6c19afb6532d69 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 14:06:47 +0200 Subject: [PATCH 12/20] fix: disable cargo term colors --- .../actions/configure-environment/action.yml | 26 +++---------------- 1 file changed, 3 insertions(+), 23 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 11e5b4b2..ce4c7e2e 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -9,9 +9,7 @@ runs: echo 'GO111MODULE=on' >> $GITHUB_ENV echo 'RUST_LOG=info' >> $GITHUB_ENV echo "GOPATH=${HOME}/go" >> $GITHUB_ENV - echo "CPATH=/usr/include:/usr/local/include:$CPATH" >> $GITHUB_ENV - echo "LIBRARY_PATH=/usr/lib:/usr/local/lib:$LIBRARY_PATH" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "CARGO_TERM_COLOR=never" >> $GITHUB_ENV shell: bash - run: | echo "/usr/local/go/bin" >> $GITHUB_PATH @@ -19,23 +17,10 @@ runs: echo "${HOME}/.cargo/bin" >> $GITHUB_PATH echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash - - if: runner.os == 'Linux' && runner.arch == 'X64' - run: | - echo "CPATH=/usr/include/x86_64-linux-gnu:$CPATH" >> $GITHUB_ENV - echo "LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LIBRARY_PATH" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH" >> $GITHUB_ENV - shell: bash - - if: runner.os == 'Linux' && runner.arch == 'ARM64' - run: | - echo "CPATH=/usr/include/aarch64-linux-gnu:$CPATH" >> $GITHUB_ENV - echo "LIBRARY_PATH=/usr/lib/aarch64-linux-gnu:$LIBRARY_PATH" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu:$LD_LIBRARY_PATH" >> $GITHUB_ENV - shell: bash - if: runner.os == 'macOS' run: | - echo "CPATH=$(brew --prefix)/include:$CPATH" >> $GITHUB_ENV - echo "LIBRARY_PATH=$(brew --prefix)/lib:$LIBRARY_PATH" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=$(brew --prefix)/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV shell: bash - if: runner.os == 'Linux' run: | @@ -46,11 +31,6 @@ runs: sudo update-alternatives --set c++ /usr/bin/g++-10 # Check if we need to install cuda-toolkit-12-5 shell: bash - - if: runner.os == 'Linux' && runner.arch == 'ARM64' - run: | - ls /usr/include/aarch64-linux-gnu - ls /usr/lib/aarch64-linux-gnu - shell: bash - if: runner.os == 'macOS' run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc From f1ad9f67fc96374a5166ef700aa7b610f498453d Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 14:32:28 +0200 Subject: [PATCH 13/20] fix: update library path on arm too --- .../actions/configure-environment/action.yml | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index ce4c7e2e..50c0cdd3 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -4,6 +4,18 @@ description: Configure environment variables for Filecoin FFI runs: using: 'composite' steps: + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin + sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 + wget -q https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo dpkg -i cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ + sudo apt-get update + sudo apt-get -y install cuda-toolkit-12-5 + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so.1 + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so + shell: bash - run: | echo "FIL_PROOFS_PARAMETER_CACHE=${GITHUB_WORKSPACE}/filecoin-proof-parameters/" >> $GITHUB_ENV echo 'GO111MODULE=on' >> $GITHUB_ENV @@ -17,6 +29,11 @@ runs: echo "${HOME}/.cargo/bin" >> $GITHUB_PATH echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash + - if: runner.os == 'Linux' && runner.arch == 'x64' + run: | + echo "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + echo "LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LIBRARY_PATH}" >> $GITHUB_ENV + shell: bash - if: runner.os == 'macOS' run: | echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV @@ -35,20 +52,6 @@ runs: run: | HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc shell: bash - - if: runner.os == 'Linux' && runner.arch == 'ARM64' - run: | - wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin - sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 - wget -q https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb - sudo dpkg -i cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb - sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ - sudo apt-get update - sudo apt-get -y install cuda-toolkit-12-5 - echo "LD_LIBRARY_PATH=/usr/local/cuda-12.5/lib64:/usr/local/cuda-12.5/lib64/stubs:$LD_LIBRARY_PATH" >> $GITHUB_ENV - sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/stubs/libcuda.so.1 - sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/libcuda.so.1 - sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/local/cuda-12.5/lib64/libcuda.so - shell: bash - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 with: toolchain: 1.73 From d3aad6d8fbb2c75b1d0a269f1af674fa738879f7 Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 15:26:27 +0200 Subject: [PATCH 14/20] fix: symlink of cuda stubs --- .github/actions/configure-environment/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 50c0cdd3..1a7848b8 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -13,6 +13,7 @@ runs: sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ sudo apt-get update sudo apt-get -y install cuda-toolkit-12-5 + sudo mkdir -p /usr/lib/aarch64-linux-gnu/stubs sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so.1 sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so shell: bash From 752f3c864f88317106096d8c3061f03a97f9c45e Mon Sep 17 00:00:00 2001 From: galargh Date: Fri, 5 Jul 2024 15:37:30 +0200 Subject: [PATCH 15/20] fix: x64 -> ARM64 --- .github/actions/configure-environment/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml index 1a7848b8..85c6c44f 100644 --- a/.github/actions/configure-environment/action.yml +++ b/.github/actions/configure-environment/action.yml @@ -30,7 +30,7 @@ runs: echo "${HOME}/.cargo/bin" >> $GITHUB_PATH echo "${HOME}/.bin" >> $GITHUB_PATH shell: bash - - if: runner.os == 'Linux' && runner.arch == 'x64' + - if: runner.os == 'Linux' && runner.arch == 'ARM64' run: | echo "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LD_LIBRARY_PATH}" >> $GITHUB_ENV echo "LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LIBRARY_PATH}" >> $GITHUB_ENV From 353fc4b9d685247bd92834286b8f7fb5c0f3c7a1 Mon Sep 17 00:00:00 2001 From: galargh Date: Sat, 6 Jul 2024 10:26:02 +0200 Subject: [PATCH 16/20] fix: do not run leak detector by default --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ee9f2e6..baae45d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,6 +17,10 @@ on: description: 'Publish the static library' required: false default: 'false' + run-leak-detector: + description: 'Run the CGO leak detector' + required: false + default: 'false' defaults: run: @@ -109,7 +113,7 @@ jobs: ./filecoin-proof-parameters - if: runner.os == 'Linux' run: cd rust && rustup target add wasm32-unknown-unknown - - if: runner.os == 'Linux' && runner.arch == 'ARM64' + - if: github.event.inputs.run-leak-detector == 'true' run: make cgo-leakdetect - if: runner.os == 'Linux' run: cd rust && FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/" RUST_LOG=info cargo test --all --release -- --test-threads 1 && cd .. From 2c7c6438c07b9485f732df9df41edb22b9804c6f Mon Sep 17 00:00:00 2001 From: galargh Date: Tue, 9 Jul 2024 12:07:52 +0100 Subject: [PATCH 17/20] chore: enabled triggering release from an arbitrary ref --- .github/workflows/ci.yml | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index baae45d6..c47afb0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,9 @@ on: description: 'Run the CGO leak detector' required: false default: 'false' + ref: + description: 'The ref to build' + required: false defaults: run: @@ -50,6 +53,11 @@ jobs: with: submodules: recursive - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} - name: Run shellcheck run: shellcheck ./install-filcrypto - name: Run cargo fmt @@ -86,6 +94,11 @@ jobs: with: submodules: recursive - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} - if: runner.os == 'macOS' run: cd rust && cargo fetch - name: Build project @@ -125,15 +138,16 @@ jobs: supraseal: name: Build with CUDA supraseal runs-on: ubuntu-latest - # # Is it OK if we operate in the default working directory? - # defaults: - # run: - # working-directory: go/src/github.com/filecoin-project/filecoin-ffi steps: - uses: actions/checkout@v4 with: submodules: recursive - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} - name: Build project with `FFI_USE_CUDA_SUPRASEAL=1` run: FFI_BUILD_FROM_SOURCE=1 FFI_USE_CUDA_SUPRASEAL=1 make publish: @@ -151,6 +165,11 @@ jobs: with: submodules: recursive - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} - if: runner.os == 'macOS' run: | cd rust && rustup target add x86_64-apple-darwin From 90e23dd4ad12f919bc5243e757315b6a67ee3b7c Mon Sep 17 00:00:00 2001 From: galargh Date: Tue, 9 Jul 2024 13:29:22 +0100 Subject: [PATCH 18/20] chore: set DEBIAN_FRONTEND env variable to noninteractive --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c47afb0e..8af8e178 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,6 +43,7 @@ env: EC_GPU_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + DEBIAN_FRONTEND: noninteractive jobs: check: @@ -138,6 +139,10 @@ jobs: supraseal: name: Build with CUDA supraseal runs-on: ubuntu-latest + # # Is it OK if we operate in the default working directory? + # defaults: + # run: + # working-directory: go/src/github.com/filecoin-project/filecoin-ffi steps: - uses: actions/checkout@v4 with: From d0ed6769a2bf440c2627cb2217b2b22119eeda24 Mon Sep 17 00:00:00 2001 From: galargh Date: Wed, 10 Jul 2024 13:50:54 +0100 Subject: [PATCH 19/20] chore: remove comments that are not needed anymore --- .github/workflows/ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8af8e178..3963cc08 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,10 +85,6 @@ jobs: matrix: runner: ['ubuntu-latest', ["self-hosted", "linux", "arm64", "xlarge"], 'macos-latest'] fail-fast: false - # # Is it OK if we operate in the default working directory? - # defaults: - # run: - # working-directory: go/src/github.com/filecoin-project/filecoin-ffi steps: - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" - uses: actions/checkout@v4 @@ -139,10 +135,6 @@ jobs: supraseal: name: Build with CUDA supraseal runs-on: ubuntu-latest - # # Is it OK if we operate in the default working directory? - # defaults: - # run: - # working-directory: go/src/github.com/filecoin-project/filecoin-ffi steps: - uses: actions/checkout@v4 with: From 0e6912348d7490f57bab251259c26cfcbc0724af Mon Sep 17 00:00:00 2001 From: Piotr Galar Date: Thu, 11 Jul 2024 15:45:35 +0200 Subject: [PATCH 20/20] Update .github/workflows/ci.yml Co-authored-by: Rod Vagg --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3963cc08..e47cce0e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: test -z "${output}" - name: Run various linters run: | - go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.56.2 + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.0 make go-lint cgo-bindings: name: Build and test CGO bindings (${{ matrix.runner }})