diff --git a/.github/actions/configure-environment/action.yml b/.github/actions/configure-environment/action.yml new file mode 100644 index 00000000..85c6c44f --- /dev/null +++ b/.github/actions/configure-environment/action.yml @@ -0,0 +1,61 @@ +name: Configure Environment Variables +description: Configure environment variables for Filecoin FFI + +runs: + using: 'composite' + steps: + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + wget -q https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-ubuntu2204.pin + sudo mv cuda-ubuntu2204.pin /etc/apt/preferences.d/cuda-repository-pin-600 + wget -q https://developer.download.nvidia.com/compute/cuda/12.5.1/local_installers/cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo dpkg -i cuda-repo-ubuntu2204-12-5-local_12.5.1-555.42.06-1_arm64.deb + sudo cp /var/cuda-repo-ubuntu2204-12-5-local/cuda-*-keyring.gpg /usr/share/keyrings/ + sudo apt-get update + sudo apt-get -y install cuda-toolkit-12-5 + sudo mkdir -p /usr/lib/aarch64-linux-gnu/stubs + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so.1 + sudo ln -s /usr/local/cuda-12.5/lib64/stubs/libcuda.so /usr/lib/aarch64-linux-gnu/stubs/libcuda.so + shell: bash + - run: | + echo "FIL_PROOFS_PARAMETER_CACHE=${GITHUB_WORKSPACE}/filecoin-proof-parameters/" >> $GITHUB_ENV + echo 'GO111MODULE=on' >> $GITHUB_ENV + echo 'RUST_LOG=info' >> $GITHUB_ENV + echo "GOPATH=${HOME}/go" >> $GITHUB_ENV + echo "CARGO_TERM_COLOR=never" >> $GITHUB_ENV + shell: bash + - run: | + echo "/usr/local/go/bin" >> $GITHUB_PATH + echo "${GOPATH}/bin" >> $GITHUB_PATH + echo "${HOME}/.cargo/bin" >> $GITHUB_PATH + echo "${HOME}/.bin" >> $GITHUB_PATH + shell: bash + - if: runner.os == 'Linux' && runner.arch == 'ARM64' + run: | + echo "LD_LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LD_LIBRARY_PATH}" >> $GITHUB_ENV + echo "LIBRARY_PATH=/usr/lib/aarch64-linux-gnu/stubs:${LIBRARY_PATH}" >> $GITHUB_ENV + shell: bash + - if: runner.os == 'macOS' + run: | + echo "CPATH=$(brew --prefix)/include" >> $GITHUB_ENV + echo "LIBRARY_PATH=$(brew --prefix)/lib" >> $GITHUB_ENV + shell: bash + - if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y valgrind ocl-icd-opencl-dev libssl-dev libhwloc-dev nvidia-cuda-toolkit g++-10 pkgconf + # Downgrade to GCC 10, as CUDA 11 doesn't play nice with GCC 11 + sudo update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 10 + sudo update-alternatives --set c++ /usr/bin/g++-10 + # Check if we need to install cuda-toolkit-12-5 + shell: bash + - if: runner.os == 'macOS' + run: | + HOMEBREW_NO_AUTO_UPDATE=1 brew install pkg-config md5sha1sum jq hwloc + shell: bash + - uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 + with: + toolchain: 1.73 + - uses: actions/setup-go@v5 + with: + go-version: '1.21' diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e47cce0e --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,213 @@ +name: CI + +on: + pull_request: + push: + branches: + - master + tags: + - v* + workflow_dispatch: + inputs: + save: + description: 'Save Filecoin parameters' + required: false + default: 'false' + publish: + description: 'Publish the static library' + required: false + default: 'false' + run-leak-detector: + description: 'Run the CGO leak detector' + required: false + default: 'false' + ref: + description: 'The ref to build' + required: false + +defaults: + run: + shell: bash + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +permissions: + contents: read + +# Can we apply these to the entire workflow? +env: + # Build the kernel only for the single architecture. This should reduce + # the overall compile-time significantly. + EC_GPU_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + BELLMAN_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + NEPTUNE_CUDA_NVCC_ARGS: --fatbin --gpu-architecture=sm_75 --generate-code=arch=compute_75,code=sm_75 + DEBIAN_FRONTEND: noninteractive + +jobs: + check: + name: Check code style and run linters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} + - name: Run shellcheck + run: shellcheck ./install-filcrypto + - name: Run cargo fmt + run: | + rustup component add rustfmt + cargo fmt --manifest-path ./rust/Cargo.toml --all -- --check + - name: Run cargo clippy + run: cd rust && cargo clippy --all-targets --features blst-portable,opencl -- -D warnings + - name: Run go fmt + # `! go fmt ./... 2>&1 | read"` doesn't work, this one does, thanks + # https://carsonip.me/posts/go-fmt-and-ci/ + run: | + output=$(go fmt ./...) + echo "${output}" + test -z "${output}" + - name: Run various linters + run: | + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.59.0 + make go-lint + cgo-bindings: + name: Build and test CGO bindings (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['ubuntu-latest', ["self-hosted", "linux", "arm64", "xlarge"], 'macos-latest'] + fail-fast: false + steps: + - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} + - if: runner.os == 'macOS' + run: cd rust && cargo fetch + - name: Build project + run: make + - name: Build project without CGO + run: env CGO_ENABLED=0 go build . + - if: runner.os == 'Linux' + uses: actions/cache/restore@v3 + with: + key: v28-proof-params-${{ runner.os }}-${{ runner.arch }} + path: | + ./filecoin-proof-parameters + - if: runner.os == 'Linux' + name: Obtain Filecoin parameters + run: | + DIR=$(pwd) + cd $(mktemp -d) + go install github.com/filecoin-project/go-paramfetch/paramfetch@latest + $GOPATH/bin/paramfetch 2048 "${DIR}/parameters.json" "${DIR}/srs-inner-product.json" + - if: runner.os == 'Linux' && ((github.event == 'push' && !startsWith(github.ref, 'refs/tags/')) || github.event.inputs.save == 'true') + uses: actions/cache/save@v3 + with: + key: v28-proof-params-${{ runner.os }}-${{ runner.arch }} + path: | + ./filecoin-proof-parameters + - if: runner.os == 'Linux' + run: cd rust && rustup target add wasm32-unknown-unknown + - if: github.event.inputs.run-leak-detector == 'true' + run: make cgo-leakdetect + - if: runner.os == 'Linux' + run: cd rust && FIL_PROOFS_PARAMETER_CACHE="${GITHUB_WORKSPACE}/filecoin-proof-parameters/" RUST_LOG=info cargo test --all --release -- --test-threads 1 && cd .. + - if: runner.os == 'Linux' + run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -p 1 -timeout 60m + - if: runner.os == 'macOS' + name: Build project and tests, but don't actually run the tests (used to verify that build/link works with Darwin) + run: GOEXPERIMENT=cgocheck2 RUST_LOG=info go test -run=^$ + supraseal: + name: Build with CUDA supraseal + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} + - name: Build project with `FFI_USE_CUDA_SUPRASEAL=1` + run: FFI_BUILD_FROM_SOURCE=1 FFI_USE_CUDA_SUPRASEAL=1 make + publish: + needs: [check, cgo-bindings, supraseal] + if: (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) || github.event.inputs.publish == 'true' + name: Publish the static library (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + strategy: + matrix: + runner: ['ubuntu-latest', ['self-hosted', 'linux', 'arm64', 'xlarge'], 'macos-latest'] + fail-fast: false + steps: + - run: echo "Running on $RUNNER_OS $RUNNER_ARCH" + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: ./.github/actions/configure-environment + - if: github.event.inputs.ref != '' + uses: actions/checkout@v4 + with: + submodules: recursive + ref: ${{ github.event.inputs.ref }} + - if: runner.os == 'macOS' + run: | + cd rust && rustup target add x86_64-apple-darwin + cd rust && cargo fetch + - if: runner.os == 'Linux' + name: Build and publish the standard release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + TARBALL_PATH="/tmp/${REPOSITORY_NAME}-$(uname)-$(uname -m)-standard.tar.gz" + RELEASE_NAME="${REPOSITORY_NAME}-$(uname)-$(uname -m)-standard" + + # Note: the blst dependency uses the portable configuration for maximum compatibility + ./scripts/build-release.sh build --verbose --no-default-features --features multicore-sdr,opencl,blst-portable + ./scripts/package-release.sh $TARBALL_PATH + ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME + - if: runner.os == 'Linux' + name: Build the optimized release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + TARBALL_PATH="/tmp/${CIRCLE_PROJECT_REPONAME}-$(uname)-$(uname -m)-optimized.tar.gz" + RUSTFLAGS="-C target-feature=$(cat rustc-target-features-optimized.json | jq -r '.[].rustc_target_feature' | tr '\n' ',')" + + ./scripts/build-release.sh build --verbose --no-default-features --features multicore-sdr,opencl + ./scripts/package-release.sh $TARBALL_PATH + - if: runner.os == 'macOS' + name: Build and publish the universal standard release + run: | + cd rust + + REPOSITORY_NAME=${GITHUB_REPOSITORY##*/} + + RELEASE_NAME="${CIRCLE_PROJECT_REPONAME}-$(uname)-standard" + TARBALL_PATH="/tmp/${RELEASE_NAME}.tar.gz" + + # Note: the blst dependency uses the portable configuration for maximum compatibility + ./scripts/build-release.sh lipo --verbose --no-default-features --features multicore-sdr,opencl,blst-portable + ./scripts/package-release.sh $TARBALL_PATH + ./scripts/publish-release.sh $TARBALL_PATH $RELEASE_NAME diff --git a/bls_test.go b/bls_test.go index c013803a..13634258 100644 --- a/bls_test.go +++ b/bls_test.go @@ -121,12 +121,10 @@ func BenchmarkBLSVerifyBatch(b *testing.B) { func benchmarkBLSVerifyBatchSize(size int) func(b *testing.B) { return func(b *testing.B) { var digests []Digest - var msgs []Message var sigs []Signature var pubks []PublicKey for i := 0; i < size; i++ { msg := Message(fmt.Sprintf("cats cats cats cats %d %d %d dogs", i, i, i)) - msgs = append(msgs, msg) digests = append(digests, Hash(msg)) priv := PrivateKeyGenerate() sig := PrivateKeySign(priv, msg)