From 956d761b02a4aab6224bc8b0209002b31abf3c66 Mon Sep 17 00:00:00 2001 From: Mike Rostecki Date: Wed, 4 Oct 2023 12:15:40 +0100 Subject: [PATCH] chore: Build bpf-linker and LLVM for multiple targets Apart from the x86_64 Linux build, cross compile also to Linux arm64, macOS x86_64 and macoOS arm64. --- .github/workflows/ci.yml | 58 +++++++++++++++++++++++++++-------- .github/workflows/llvm.yml | 49 ++++++++++++++++++++++++----- .github/workflows/release.yml | 25 +++++++++++++-- 3 files changed, 108 insertions(+), 24 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86c83cb0..dd3481a7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -48,7 +48,6 @@ jobs: run: cargo fmt --all -- --check build: - runs-on: ubuntu-22.04 strategy: fail-fast: false matrix: @@ -59,7 +58,21 @@ jobs: llvm: - 17 - source - name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} + include: + - target: aarch64-unknown-linux-musl + llvm-target: aarch64-linux-gnu + os: ubuntu-latest + - target: aarch64-apple-darwin + llvm-target: aarch64-apple-darwin + os: macos-latest + - target: x86_64-unknown-linux-musl + llvm-target: x86_64-linux-gnu + os: ubuntu-latest + - target: x86_64-apple-darwin + llvm-target: x86_64-apple-darwin + os: macos-latest + name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} target=${{ matrix.target }} + runs-on: ${{ matrix.os }} needs: llvm env: @@ -85,14 +98,19 @@ jobs: - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/setup-cross-toolchain-action@v1 + if: startsWith(matrix.os, 'ubuntu') + with: + target: ${{ matrix.target }} + - name: Check (default features, no system LLVM) run: cargo check - name: Build (default features, no system LLVM) run: cargo build - - name: Install dependencies - if: matrix.rust == 'nightly' + - name: Install dependencies (Linux) + if: matrix.rust == 'nightly' && startsWith(matrix.os, 'ubuntu') # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit # enum values which was added in clang 15[1]. # @@ -110,8 +128,8 @@ jobs: sudo apt update sudo apt -y install clang gcc-multilib - - name: Install LLVM - if: matrix.llvm != 'source' + - name: Install LLVM (Linux) + if: matrix.llvm != 'source' && startsWith(matrix.os, 'ubuntu') run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -121,12 +139,22 @@ jobs: sudo apt -y install llvm-${{ matrix.llvm }}-dev echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH + - name: Install LLVM (macOS) + if: matrix.llvm != 'source' && startsWith(matrix.os, 'macos') + run: | + set -euxo pipefail + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo "/usr/local/bin/brew" >> $GITHUB_PATH + brew install llvm + echo "$(brew --prefix llvm)/bin" >> $GITHUB_PATH + echo "LLVM_SYS_170_PREFIX=$(brew --prefix llvm)" >> $GITHUB_ENV + - name: Restore LLVM if: matrix.llvm == 'source' uses: actions/cache/restore@v3 with: path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.llvm-target)] }} fail-on-cache-miss: true - name: Add LLVM to PATH && LD_LIBRARY_PATH @@ -151,6 +179,10 @@ jobs: - name: Check run: cargo hack check --feature-powerset --features llvm-sys/force-dynamic + - name: Setup tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + - name: Build run: cargo hack build --feature-powerset --features llvm-sys/force-dynamic @@ -159,19 +191,19 @@ jobs: run: cargo hack test --feature-powerset --features llvm-sys/force-dynamic - uses: actions/checkout@v4 - if: matrix.rust == 'nightly' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' with: repository: aya-rs/aya path: aya submodules: recursive - name: Install - if: matrix.rust == 'nightly' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' run: cargo install --path . --no-default-features --features llvm-sys/force-dynamic # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. - name: Download debian kernels - if: matrix.rust == 'nightly' && runner.arch == 'ARM64' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' && runner.arch == 'ARM64' working-directory: aya run: | set -euxo pipefail @@ -182,7 +214,7 @@ jobs: # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. - name: Download debian kernels - if: matrix.rust == 'nightly' && runner.arch == 'X64' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' && runner.arch == 'X64' working-directory: aya run: | set -euxo pipefail @@ -193,7 +225,7 @@ jobs: # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. - name: Extract debian kernels - if: matrix.rust == 'nightly' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' working-directory: aya run: | set -euxo pipefail @@ -201,7 +233,7 @@ jobs: sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -" - name: Run aya integration tests - if: matrix.rust == 'nightly' + if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' working-directory: aya run: | set -euxo pipefail diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 0fd2819a..d14d65a8 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -2,13 +2,32 @@ name: LLVM on: workflow_call: + # NOTE(vadorovsky): Unfortunately, Github Actions don't support matrix + # strategy in `on` section. So we have to specify outputs by hand. outputs: - cache-key: - value: ${{ jobs.llvm.outputs.cache-key }} + cache-key-aarch64-linux-gnu: + value: ${{ jobs.llvm.outputs.cache-key-aarch64-linux-gnu }} + cache-key-aarch64-apple-darwin: + value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }} + cache-key-x86_64-linux-gnu: + value: ${{ jobs.llvm.outputs.cache-key-x86_64-linux-gnu }} + cache-key-x86_64-apple-darwin: + value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }} jobs: llvm: - runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: aarch64-linux-gnu + os: ubuntu-latest + - target: aarch64-apple-darwin + os: macos-latest + - target: x86_64-linux-gnu + os: ubuntu-latest + - target: x86_64-apple-darwin + os: macos-latest + runs-on: ${{ matrix.os }} name: llvm outputs: cache-key: ${{ steps.cache-key.outputs.cache-key }} @@ -20,18 +39,18 @@ jobs: echo "sha=$value" >> "$GITHUB_OUTPUT" - id: cache-key - run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" + run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" - name: Cache id: cache-llvm uses: actions/cache@v3 with: path: llvm-install - key: ${{ steps.cache-key.outputs.cache-key }} + key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }} lookup-only: true - - name: Install Tools - if: steps.cache-llvm.outputs.cache-hit != 'true' + - name: Install Tools (Linux) + if: steps.cache-llvm.outputs.cache-hit != 'true' && startsWith(matrix.os, 'ubuntu') run: | set -euxo pipefail wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ @@ -43,6 +62,14 @@ jobs: sudo apt update sudo apt -y install cmake ninja-build clang lld + - name: Install Tools (macOS) + if: steps.cache-llvm.outputs.cache-hit != 'true' && startsWith(matrix.os, 'macos-latest') + run: | + set -euxo pipefail + /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + echo "/usr/local/bin/brew" >> $GITHUB_PATH + brew install cmake ninja + - name: Checkout LLVM Source if: steps.cache-llvm.outputs.cache-hit != 'true' uses: actions/checkout@v4 @@ -55,6 +82,10 @@ jobs: if: steps.cache-llvm.outputs.cache-hit != 'true' run: | set -euxo pipefail + LLVM_USE_LINKER_OPTION="" + if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then + LLVM_USE_LINKER_OPTION="-DLLVM_USE_LINKER=lld" + fi cmake \ -S llvm-project/llvm \ -B llvm-build \ @@ -64,13 +95,15 @@ jobs: -DCMAKE_CXX_COMPILER=clang++ \ -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \ -DLLVM_BUILD_LLVM_DYLIB=ON \ + -DLLVM_DEFAULT_TARGET_TRIPLE=${{ matrix.target }} \ -DLLVM_ENABLE_ASSERTIONS=ON \ -DLLVM_ENABLE_PROJECTS= \ -DLLVM_ENABLE_RUNTIMES= \ + -DLLVM_HOST_TRIPLE=${{ matrix.target }} \ -DLLVM_INSTALL_UTILS=ON \ -DLLVM_LINK_LLVM_DYLIB=ON \ -DLLVM_TARGETS_TO_BUILD=BPF \ - -DLLVM_USE_LINKER=lld + $LLVM_USE_LINKER_OPTION - name: Install LLVM if: steps.cache-llvm.outputs.cache-hit != 'true' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 180be024..9b6de584 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,15 +9,29 @@ jobs: uses: ./.github/workflows/llvm.yml upload-bins: - # TODO: Build for macos someday. - runs-on: ubuntu-22.04 + strategy: + matrix: + include: + - target: aarch64-unknown-linux-musl + llvm-target: aarch64-linux-gnu + os: ubuntu-latest + - target: aarch64-apple-darwin + llvm-target: aarch64-apple-darwin + os: macos-latest + - target: x86_64-unknown-linux-musl + llvm-target: x86_64-linux-gnu + os: ubuntu-latest + - target: x86_64-apple-darwin + llvm-target: x86_64-apple-darwin + os: macos-latest + runs-on: ${{ matrix.os }} needs: llvm steps: - name: Restore LLVM uses: actions/cache/restore@v3 with: path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.llvm-target)] }} fail-on-cache-miss: true - name: Add LLVM to PATH @@ -28,9 +42,14 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/setup-cross-toolchain-action@v1 + if: startsWith(matrix.os, 'ubuntu') + with: + target: ${{ matrix.target }} - uses: taiki-e/upload-rust-binary-action@v1 with: bin: bpf-linker + target: ${{ matrix.target }} features: llvm-sys/force-static env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}