From 12901022066753b4575d855500dcc23f7ac12ed4 Mon Sep 17 00:00:00 2001 From: Michal Rostecki Date: Tue, 13 Feb 2024 20:17:57 +0100 Subject: [PATCH] chore: Build bpf-linker and LLVM for multiple targets Apart from the x86_64 Linux build, cross compile also to Linux arm64, macOS x86_64 and macoOS arm64. Use musl as libc for Linux release builds. cargo-binstall falls back to musl artifacts even for *-gnu targets in Rust. --- .github/actions/build-llvm/action.yml | 121 +++++++++ .github/workflows/ci.yml | 337 ++++++++++++++++++++++++-- .github/workflows/llvm.yml | 170 ++++++++----- .github/workflows/release.yml | 84 ++++++- tests/tests.rs | 73 +++++- 5 files changed, 698 insertions(+), 87 deletions(-) create mode 100644 .github/actions/build-llvm/action.yml diff --git a/.github/actions/build-llvm/action.yml b/.github/actions/build-llvm/action.yml new file mode 100644 index 00000000..67d4cd4a --- /dev/null +++ b/.github/actions/build-llvm/action.yml @@ -0,0 +1,121 @@ +description: Build LLVM + +inputs: + git-ref: + description: "LLVM git repository reference" + required: true + cxxflags: + description: "Additional CXXFLAGS" + required: false + ldflags: + description: "Additional LDFLAGS" + required: false + clang: + description: "Name of clang binary" + required: true + clangxx: + description: "Name of clang++ binary" + required: true + find-root-path-mode-program: + description: "Whether to search for programs in CMAKE_FIND_ROOT_PATH" + required: false + system-name: + description: "Operating system type" + required: true + system-processor: + description: "Processor family" + required: true + sysroot: + description: "Root directory for the given architecture" + required: false + build-static: + description: "Link dependencies statically" + required: true + enable-libcxx: + description: "Enable LLVM libc++ as the standard library" + required: true + host-triple: + description: "Host triple" + required: true + linker: + description: "Linker" + required: false + install-dir: + description: "Directory in which artifacts are installed" + +runs: + using: "composite" + steps: + - name: Checkout LLVM Source + uses: actions/checkout@v4 + with: + repository: aya-rs/llvm-project + ref: ${{ inputs.git-ref }} + path: llvm-project + + - name: Configure LLVM + shell: bash + env: + CXXFLAGS: ${{ inputs.cxxflags }} + LDFLAGS: ${{ inputs.ldflags }} + run: | + set -euxo pipefail + cmake \ + -S llvm-project/llvm \ + -B llvm-build \ + -G Ninja \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DCMAKE_ASM_COMPILER="${{ inputs.clang }}" \ + -DCMAKE_C_COMPILER="${{ inputs.clang }}" \ + -DCMAKE_CXX_COMPILER="${{ inputs.clangxx }}" \ + -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM="${{ inputs.find-root-path-mode-program }}" \ + -DCMAKE_INSTALL_PREFIX="${{ inputs.install-dir }}" \ + -DCMAKE_SKIP_INSTALL_RPATH="${{ inputs.build-static }}" \ + -DCMAKE_SYSROOT="${{ inputs.sysroot }}" \ + -DCMAKE_SYSTEM_NAME="${{ inputs.system-name }}" \ + -DCMAKE_SYSTEM_PROCESSOR="${{ inputs.system-processor }}" \ + -DLLVM_BUILD_STATIC="${{ inputs.build-static }}" \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_ENABLE_ASSERTIONS=ON \ + -DLLVM_ENABLE_LIBCXX="${{ inputs.enable-libcxx }}" \ + -DLLVM_ENABLE_PROJECTS= \ + -DLLVM_ENABLE_RUNTIMES= \ + -DLLVM_HOST_TRIPLE="${{ inputs.host-triple }}" \ + -DLLVM_USE_LINKER="${{ inputs.linker }}" \ + -DLLVM_USE_STATIC_ZSTD=ON \ + -DLLVM_TARGETS_TO_BUILD=BPF + + - name: Build LLVM + # env: + # # Create symlinks rather than copies to conserve disk space. At the time of this writing, + # # GitHub-hosted runners have 14GB of SSD space + # # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources). + # # + # # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those + # # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases. + # CMAKE_INSTALL_MODE: ABS_SYMLINK + shell: bash + run: | + set -euxo pipefail + cmake --build llvm-build \ + --target llvm-config \ + --target install + # There is no target for **installing** the native llvm-config build, even + # though a standalone build is possible. Let's just move it. + mv llvm-build/NATIVE/bin/llvm-config ${{ inputs.install-dir }}/bin/ + + # - name: Rewrite LLVM Symlinks + # # Move targets over the symlinks that point to them. + # # + # # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE. + # shell: bash + # run: | + # set -euxo pipefail + # find llvm-install -type l -execdir sh -eux -c ' + # for link in "$@"; do + # target=$(readlink "$link") + # case $target in + # /*) mv "$target" "$link" ;; + # esac + # done + # ' sh {} + diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b9842f4b..455bb551 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,8 +47,8 @@ jobs: - name: Check formatting run: cargo fmt --all -- --check - build: - runs-on: ubuntu-22.04 + # A simple, native build on Ubuntu. + build-linux-native: strategy: fail-fast: false matrix: @@ -61,8 +61,8 @@ jobs: - nightly-2024-04-16 llvm: - 18 - - source - name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} + name: native rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} + runs-on: ubuntu-22.04 needs: llvm env: @@ -71,7 +71,20 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Rust ${{ matrix.rust }} + - name: Install dependencies + shell: bash + run: | + sudo apt update + sudo apt -y install \ + build-essential \ + curl \ + libzstd-dev \ + sudo \ + wget \ + zlib1g-dev \ + zstd + + - name: Install Rust ${{ matrix.target.rust }} if: matrix.rust != 'nightly' uses: dtolnay/rust-toolchain@master with: @@ -86,6 +99,16 @@ jobs: # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl + - name: Install bpftool + if: matrix.rust == 'nightly' + shell: bash + env: + BPFTOOL_VERSION: 7.3.0 + run: | + wget "https://github.com/libbpf/bpftool/releases/download/v${BPFTOOL_VERSION}/bpftool-v${BPFTOOL_VERSION}-amd64.tar.gz" + sudo tar xpf "bpftool-v${BPFTOOL_VERSION}-amd64.tar.gz" -C /usr/bin + sudo chmod +x /usr/bin/bpftool + - uses: Swatinem/rust-cache@v2 - name: Check (default features, no system LLVM) @@ -106,6 +129,7 @@ jobs: # [0] https://github.com/actions/runner-images/blob/ubuntu22/20230724.1/images/linux/Ubuntu2204-Readme.md # # [1] https://github.com/llvm/llvm-project/commit/dc1c43d + shell: bash run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc @@ -115,25 +139,27 @@ jobs: - name: Install LLVM if: matrix.llvm != 'source' + shell: bash run: | set -euxo pipefail wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc echo -e deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-${{ matrix.llvm }} main | sudo tee /etc/apt/sources.list.d/llvm.list sudo apt update - sudo apt -y install llvm-${{ matrix.llvm }}-dev + sudo apt -y install llvm-${{ matrix.llvm }}-dev libpolly-${{ matrix.llvm }}-dev echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH - name: Restore LLVM if: matrix.llvm == 'source' uses: actions/cache/restore@v4 with: - path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + path: ${{ github.workspace }}/llvm-install + key: ${{ needs.llvm.outputs['cache-key-llvm-x86_64-unknown-linux-musl'] }} fail-on-cache-miss: true - name: Add LLVM to PATH && LD_LIBRARY_PATH if: matrix.llvm == 'source' + shell: bash run: | set -euxo pipefail echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH @@ -144,6 +170,7 @@ jobs: # llvm-sys discovers link flags at build script time; these are cached by cargo. The cached # flags may be incorrect when the cache is reused across LLVM versions. - name: Bust llvm-sys cache + shell: bash run: | set -euxo pipefail cargo clean -p llvm-sys @@ -152,25 +179,26 @@ jobs: - uses: taiki-e/install-action@cargo-hack - name: Check - run: cargo hack check --feature-powerset --features llvm-sys/force-dynamic + run: cargo hack check --feature-powerset - name: Build - run: cargo hack build --feature-powerset --features llvm-sys/force-dynamic + run: cargo hack build --feature-powerset - name: Test if: matrix.rust == 'nightly' - run: cargo hack test --feature-powerset --features llvm-sys/force-dynamic + run: cargo hack test --feature-powerset - uses: actions/checkout@v4 if: matrix.rust == 'nightly' with: - repository: aya-rs/aya + repository: vadorovsky/aya path: aya + ref: integration-tests-linker submodules: recursive - name: Install if: matrix.rust == 'nightly' - run: cargo install --path . --no-default-features --features llvm-sys/force-dynamic + run: cargo install --path . --no-default-features # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. - name: Download debian kernels @@ -178,7 +206,7 @@ jobs: working-directory: aya run: | set -euxo pipefail - mkdir -p test/.tmp/debian-kernels/arm64 + mkdir -p test/.tmp/debian-kernels/arm64 printf '%s\0' \ linux-image-6.1.0-15-cloud-arm64-unsigned_6.1.66-1_arm64.deb \ | xargs -0 -t -P0 -I {} wget -nd -nv -P test/.tmp/debian-kernels/arm64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{} @@ -210,3 +238,284 @@ jobs: set -euxo pipefail sudo apt install -y locate qemu-system-{arm,x86} find test/.tmp -name 'vmlinuz-*' | RUSTFLAGS=-Cdebuginfo=line-directives-only xargs -t cargo xtask integration-test vm + + # A cross build on Gentoo with musl-llvm profile (a combo which @vadorovsky + # finds as the most painless for cross build environments with static linking 😉). + # We use only our LLVM builds here. It's mostly for us to ensure that we can + # build binstall tarballs. + build-linux-cross: + strategy: + fail-fast: false + matrix: + rust: + - stable + - beta + - nightly + target: + - target: aarch64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains-cross-arm64 + sysroot: /usr/aarch64-gentoo-linux-musl + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld \ + -C link-arg=--target=aarch64-gentoo-linux-musl \ + -C link-arg=--sysroot=/usr/aarch64-gentoo-linux-musl \ + -L /usr/aarch64-gentoo-linux-musl/lib \ + -L /usr/aarch64-gentoo-linux-musl/usr/lib \ + -L /usr/aarch64-gentoo-linux-musl/usr/lib/clang/17/lib \ + -l static=c++abi" + - target: x86_64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains + sysroot: / + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld \ + -L /lib -L /usr/lib -L /usr/lib/clang/17/lib \ + -l static=c++abi" + name: cross rustc=${{ matrix.rust }} target=${{ matrix.target.target }} + runs-on: ubuntu-latest + container: + image: ${{ matrix.target.container }} + needs: llvm + + env: + RUST_BACKTRACE: full + + steps: + - name: Save space + run: | + rm -rf /__t/{CodeQL,PyPy,Python,Ruby,go} + + - uses: actions/checkout@v4 + + - name: Add system LLVM toolchain (clang, lld) to PATH + shell: bash + run: | + set -euxo pipefail + echo "/usr/lib/llvm/17/bin" >> $GITHUB_PATH + + # NOTE(vadorovsky): In Gentoo, /usr/lib/libc++{.a,so} are ldscripts. + # However, Rust expects an actual static library there. A long-term + # solution would be teaching build.rs in llvm-sys to detect that, but for + # now let's just move the static lib to the expected location. + - name: Move libc++ static lib to libc++.a + shell: bash + run: | + mv ${{ matrix.target.sysroot }}/usr/lib/libc++_static.a ${{ matrix.target.sysroot }}/usr/lib/libc++.a + + - name: Install Rust ${{ matrix.target.rust }} + if: matrix.rust != 'nightly' + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Install Rust ${{ matrix.rust }} + if: matrix.rust == 'nightly' + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + components: rust-src + # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. + targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl + + - uses: Swatinem/rust-cache@v2 + + - name: Setup Rust target + run: | + rustup +${{ matrix.rust }} target add ${{ matrix.target.target }} + + - name: Check (default features, no system LLVM) + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: cargo check --target ${{ matrix.target.target }} + + - name: Build (default features, no system LLVM) + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: cargo build --target ${{ matrix.target.target }} + + - name: Restore LLVM + uses: actions/cache/restore@v4 + with: + path: ${{ env.GITHUB_WORKSPACE }}/llvm-install + key: ${{ needs.llvm.outputs[format('cache-key-llvm-{0}', matrix.target.target)] }} + fail-on-cache-miss: true + + - name: Add our LLVM build to PATH && LD_LIBRARY_PATH + shell: bash + run: | + set -euxo pipefail + pwd + echo "${{ env.GITHUB_WORKSPACE }}/llvm-install/bin" >> $GITHUB_PATH + # LD_LIBRARY_PATH is needed because we're going to link everything dynamically below. This + # doesn't affect behavior, but greatly reduces disk usage. + echo "LD_LIBRARY_PATH=${{ env.GITHUB_WORKSPACE }}/llvm-install/lib" >> $GITHUB_ENV + + # llvm-sys discovers link flags at build script time; these are cached by cargo. The cached + # flags may be incorrect when the cache is reused across LLVM versions. + - name: Bust llvm-sys cache + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + shell: bash + run: | + set -euxo pipefail + cargo clean -p llvm-sys + cargo clean -p llvm-sys --release + + - name: Install cargo-hack + run: | + cargo install cargo-hack + + - name: Check + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: | + cargo hack check --feature-powerset --target ${{ matrix.target.target }} + + - name: Build + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: cargo hack build --feature-powerset --target ${{ matrix.target.target }} + + - name: Test + if: matrix.rust == 'nightly' + env: + CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_RUNNER: qemu-aarch64 + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: cargo test --no-default-features --features llvm-sys/force-static --target ${{ matrix.target.target }} + + - uses: actions/checkout@v4 + if: matrix.rust == 'nightly' + with: + repository: vadorovsky/aya + path: aya + ref: integration-tests-linker + submodules: recursive + + - name: Install + if: matrix.rust == 'nightly' + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: cargo install --path . --no-default-features --features llvm-sys/force-static --target ${{ matrix.target.target }} + + - name: Save some space + run: | + rm -rf /usr/aarch64-gentoo-linux-musl + + # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. + - name: Download debian kernels + if: matrix.rust == 'nightly' && matrix.target.target == 'x86_64-unknown-linux-musl' + working-directory: aya + shell: bash + run: | + set -euxo pipefail + mkdir -p test/.tmp/debian-kernels/amd64 + printf '%s\0' \ + linux-image-6.1.0-15-cloud-amd64-unsigned_6.1.66-1_amd64.deb \ + | xargs -0 -t -P0 -I {} wget -nd -nv -P test/.tmp/debian-kernels/amd64 ftp://ftp.us.debian.org/debian/pool/main/l/linux/{} + + # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. + - name: Extract debian kernels + if: matrix.rust == 'nightly' && matrix.target.target == 'x86_64-unknown-linux-musl' + working-directory: aya + shell: bash + run: | + set -euxo pipefail + pushd test/.tmp + find . -name '*.deb' -print0 | while IFS= read -r -d $'\0' debfile; do + ar x "$debfile" + tar xf data.tar.* --wildcards '*vmlinuz*' + done + + - name: Run aya integration tests + if: matrix.rust == 'nightly' && matrix.target.target == 'x86_64-unknown-linux-musl' + working-directory: aya + shell: bash + env: + RUSTFLAGS: ${{ matrix.target.rustflags }} + run: | + set -euxo pipefail + find test/.tmp -name 'vmlinuz-*' | RUSTFLAGS=-Cdebuginfo=line-directives-only xargs -t cargo xtask integration-test vm + + build-macos: + strategy: + fail-fast: false + matrix: + rust: + - stable + - beta + - nightly + target: + - target: aarch64-apple-darwin + - target: x86_64-apple-darwin + name: rustc=${{ matrix.rust }} target=${{ matrix.target.target }} + runs-on: macos-latest + needs: llvm + + env: + RUST_BACKTRACE: full + + steps: + - uses: actions/checkout@v4 + + - name: Install dependencies + run: | + set -euxo pipefail + brew install llvm + echo /usr/local/opt/llvm/bin/ >> $GITHUB_PATH + + - name: Install Rust ${{ matrix.rust }} + if: matrix.rust != 'nightly' + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + + - name: Install Rust ${{ matrix.rust }} + if: matrix.rust == 'nightly' + uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.rust }} + components: rust-src + # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15. + targets: aarch64-unknown-linux-musl,x86_64-unknown-linux-musl + + - uses: Swatinem/rust-cache@v2 + + - name: Check (default features, no system LLVM) + run: cargo check + + - name: Build (default features, no system LLVM) + run: cargo build + + - name: Restore LLVM + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/llvm-install + key: ${{ needs.llvm.outputs[format('cache-key-llvm-{0}', matrix.target.target)] }} + fail-on-cache-miss: true + + - name: Add LLVM to PATH && LD_LIBRARY_PATH + shell: bash + run: | + set -euxo pipefail + echo "${{ github.workspace }}/llvm-install/bin" >> $GITHUB_PATH + # LD_LIBRARY_PATH is needed because we're going to link everything dynamically below. This + # doesn't affect behavior, but greatly reduces disk usage. + echo "LD_LIBRARY_PATH=${{ github.workspace }}/llvm-install/lib" >> $GITHUB_ENV + + # llvm-sys discovers link flags at build script time; these are cached by cargo. The cached + # flags may be incorrect when the cache is reused across LLVM versions. + - name: Bust llvm-sys cache + shell: bash + run: | + set -euxo pipefail + cargo clean -p llvm-sys + cargo clean -p llvm-sys --release + + - uses: taiki-e/install-action@cargo-hack + + - name: Check + run: cargo hack check --feature-powerset + + - name: Build + run: cargo hack build --feature-powerset + + - name: Test + if: matrix.rust == 'nightly' + run: cargo hack test --feature-powerset diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml index 6fb0654a..9545d4cd 100644 --- a/.github/workflows/llvm.yml +++ b/.github/workflows/llvm.yml @@ -3,99 +3,137 @@ name: LLVM on: workflow_call: outputs: - cache-key: - value: ${{ jobs.llvm.outputs.cache-key }} + cache-key-llvm-aarch64-apple-darwin: + value: ${{ jobs.llvm-macos.outputs.cache-key-llvm-aarch64-apple-darwin }} + cache-key-llvm-aarch64-unknown-linux-musl: + value: ${{ jobs.llvm-linux.outputs.cache-key-llvm-aarch64-unknown-linux-musl }} + cache-key-llvm-x86_64-apple-darwin: + value: ${{ jobs.llvm-macos.outputs.cache-key-llvm-x86_64-apple-darwin }} + cache-key-llvm-x86_64-unknown-linux-musl: + value: ${{ jobs.llvm-linux.outputs.cache-key-llvm-x86_64-unknown-linux-musl }} jobs: - llvm: - runs-on: ubuntu-22.04 - name: llvm + llvm-linux: + strategy: + matrix: + include: + - target: aarch64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains-cross-arm64 + clang: aarch64-gentoo-linux-musl-clang + clangxx: aarch64-gentoo-linux-musl-clang++ + system-processor: aarch64 + sysroot: /usr/aarch64-gentoo-linux-musl + - target: x86_64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains + clang: clang + clangxx: clang++ + system-processor: x86_64 + sysroot: / + runs-on: ubuntu-latest + container: + image: ${{ matrix.target.container }} + name: LLVM Linux outputs: - cache-key: ${{ steps.cache-key.outputs.cache-key }} + cache-key-llvm-aarch64-unknown-linux-musl: ${{ steps.cache-key-llvm.outputs.cache-key-llvm-aarch64-unknown-linux-musl }} + cache-key-llvm-x86_64-unknown-linux-musl: ${{ steps.cache-key-llvm.outputs.cache-key-llvm-x86_64-unknown-linux-musl }} steps: - - id: ls-remote + - uses: actions/checkout@v4 + + - name: Check LLVM repository + id: ls-remote run: | set -euxo pipefail value=$(git ls-remote https://github.com/aya-rs/llvm-project.git refs/heads/rustc/18.0-2024-02-13 | cut -f1) echo "sha=$value" >> "$GITHUB_OUTPUT" - - id: cache-key - run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" + - name: Construct LLVM cache ID + id: cache-key-llvm + run: echo "cache-key-llvm-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" - name: Cache id: cache-llvm uses: actions/cache@v4 with: - path: llvm-install - key: ${{ steps.cache-key.outputs.cache-key }} + path: ${{ env.GITHUB_WORKSPACE }}/llvm-install + key: ${{ steps.cache-key-llvm.outputs[format('cache-key-llvm-{0}', matrix.target)] }} lookup-only: true - - name: Install Tools + - name: Add LLVM to PATH if: steps.cache-llvm.outputs.cache-hit != 'true' run: | set -euxo pipefail - wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \ - gpg --dearmor - | \ - sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null - echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ focal main' | \ - sudo tee /etc/apt/sources.list.d/kitware.list >/dev/null - - sudo apt update - sudo apt -y install cmake ninja-build clang lld + echo "/usr/lib/llvm/17/bin" >> $GITHUB_PATH - - name: Checkout LLVM Source + - name: Build LLVM if: steps.cache-llvm.outputs.cache-hit != 'true' - uses: actions/checkout@v4 + uses: ./.github/actions/build-llvm with: - repository: aya-rs/llvm-project - ref: ${{ steps.ls-remote.outputs.sha }} - path: llvm-project + git-ref: ${{ steps.ls-remote.outputs.sha }} + cxxflags: "-stdlib=libc++" + ldflags: "-fuse-ld=lld -rtlib=compiler-rt -unwindlib=libunwind" + clang: ${{ matrix.clang }} + clangxx: ${{ matrix.clangxx }} + find-root-path-mode-program: NEVER + system-name: Linux + system-processor: ${{ matrix.system-processor }} + sysroot: ${{ matrix.sysroot }} + build-static: ON + enable-libcxx: ON + host-triple: ${{ matrix.target }} + linker: lld + install-dir: ${{ env.GITHUB_WORKSPACE }}/llvm-install - - name: Configure LLVM - if: steps.cache-llvm.outputs.cache-hit != 'true' + llvm-macos: + strategy: + matrix: + include: + - target: aarch64-apple-darwin + system-processor: aarch64 + - target: x86_64-apple-darwin + system-processor: x86_64 + runs-on: macos-latest + name: LLVM macOS + outputs: + cache-key-llvm-aarch64-apple-darwin: ${{ steps.cache-key-llvm.outputs.cache-key-llvm-aarch64-apple-darwin }} + cache-key-llvm-x86_64-apple-darwin: ${{ steps.cache-key-llvm.outputs.cache-key-llvm-x86_64-apple-darwin }} + steps: + - uses: actions/checkout@v4 + + - name: Check LLVM repository + id: ls-remote run: | set -euxo pipefail - cmake \ - -S llvm-project/llvm \ - -B llvm-build \ - -G Ninja \ - -DCMAKE_BUILD_TYPE=RelWithDebInfo \ - -DCMAKE_C_COMPILER=clang \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \ - -DLLVM_BUILD_LLVM_DYLIB=ON \ - -DLLVM_ENABLE_ASSERTIONS=ON \ - -DLLVM_ENABLE_PROJECTS= \ - -DLLVM_ENABLE_RUNTIMES= \ - -DLLVM_INSTALL_UTILS=ON \ - -DLLVM_LINK_LLVM_DYLIB=ON \ - -DLLVM_TARGETS_TO_BUILD=BPF \ - -DLLVM_USE_LINKER=lld + value=$(git ls-remote https://github.com/aya-rs/llvm-project.git refs/heads/rustc/18.0-2024-02-13 | cut -f1) + echo "sha=$value" >> "$GITHUB_OUTPUT" - - name: Install LLVM - if: steps.cache-llvm.outputs.cache-hit != 'true' - env: - # Create symlinks rather than copies to conserve disk space. At the time of this writing, - # GitHub-hosted runners have 14GB of SSD space - # (https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources). - # - # Since the LLVM build creates a bunch of symlinks (and this setting does not turn those - # into symlinks-to-symlinks), use absolute symlinks so we can distinguish the two cases. - CMAKE_INSTALL_MODE: ABS_SYMLINK - run: cmake --build llvm-build --target install + - name: Construct LLVM cache ID + id: cache-key-llvm + run: echo "cache-key-llvm-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT" - - name: Rewrite LLVM Symlinks + - name: Cache + id: cache-llvm + uses: actions/cache@v4 + with: + path: ${{ github.workspace }}/llvm-install + key: ${{ steps.cache-key-llvm.outputs[format('cache-key-llvm-{0}', matrix.target)] }} + lookup-only: true + + - name: Install Tools (macOS) if: steps.cache-llvm.outputs.cache-hit != 'true' - # Move targets over the symlinks that point to them. - # - # This whole dance would be simpler if CMake supported CMAKE_INSTALL_MODE=MOVE. run: | set -euxo pipefail - find llvm-install -type l -execdir sh -eux -c ' - for link in "$@"; do - target=$(readlink "$link") - case $target in - /*) mv "$target" "$link" ;; - esac - done - ' sh {} + + brew install cmake ninja + + - name: Build LLVM + if: steps.cache-llvm.outputs.cache-hit != 'true' + uses: ./.github/actions/build-llvm + with: + git-ref: ${{ steps.ls-remote.outputs.sha }} + clang: clang + clangxx: clang++ + system-name: Darwin + system-processor: ${{ matrix.system-processor }} + build-static: OFF + enable-libcxx: OFF + host-triple: ${{ matrix.target }} + install-dir: ${{ github.workspace }}/llvm-install diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1cb4c1c3..7843c804 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,16 +8,89 @@ jobs: llvm: uses: ./.github/workflows/llvm.yml - upload-bins: - # TODO: Build for macos someday. - runs-on: ubuntu-22.04 + upload-bins-linux: + strategy: + matrix: + include: + - target: aarch64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains-cross-arm64 + sysroot: /usr/aarch64-gentoo-linux-musl + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld \ + -C link-arg=--target=aarch64-gentoo-linux-musl \ + -C link-arg=--sysroot=/usr/aarch64-gentoo-linux-musl \ + -L /usr/aarch64-gentoo-linux-musl/lib \ + -L /usr/aarch64-gentoo-linux-musl/usr/lib \ + -L /usr/aarch64-gentoo-linux-musl/usr/lib/clang/17/lib \ + -l static=c++abi" + - target: x86_64-unknown-linux-musl + container: gitlab.com/vadorovsky/gentoo-musl-llvm:stage4-amd64-toolchains + sysroot: / + rustflags: "-C linker=clang -C link-arg=-fuse-ld=lld \ + -L /lib -L /usr/lib -L /usr/lib/clang/17/lib \ + -l static=c++abi" + runs-on: ubuntu-latest + container: + image: ${{ matrix.container }} needs: llvm steps: - name: Restore LLVM uses: actions/cache/restore@v4 with: - path: llvm-install - key: ${{ needs.llvm.outputs.cache-key }} + path: ${{ env.GITHUB_WORKSPACE }}/llvm-install + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target)] }} + fail-on-cache-miss: true + + - name: Add LLVM to PATH + run: | + echo "${{ env.GITHUB_WORKSPACE }}/llvm-install/bin" >> $GITHUB_PATH + echo "$PATH" + + - uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + + - name: Add system LLVM toolchain (clang, lld) to PATH + shell: bash + run: | + set -euxo pipefail + echo "/usr/lib/llvm/17/bin" >> $GITHUB_PATH + + # NOTE(vadorovsky): In Gentoo, /usr/lib/libc++{.a,so} are ldscripts. + # However, Rust expects an actual static library there. A long-term + # solution would be teaching build.rs in llvm-sys to detect that, but for + # now let's just move the static lib to the expected location. + - name: Move libc++ static lib to libc++.a + shell: bash + run: | + mv ${{ matrix.sysroot }}/usr/lib/libc++_static.a ${{ matrix.sysroot }}/usr/lib/libc++.a + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + + - uses: taiki-e/upload-rust-binary-action@v1 + with: + bin: bpf-linker + target: ${{ matrix.target }} + features: llvm-sys/force-static + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUSTFLAGS: ${{ matrix.rustflags }} + + upload-bins-macos: + strategy: + matrix: + include: + - target: aarch64-apple-darwin + - target: x86_64-apple-darwin + runs-on: macos-latest + needs: llvm + steps: + - name: Restore LLVM + uses: actions/cache/restore@v4 + with: + path: ${{ github.workspace }}/llvm-install + key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.target)] }} fail-on-cache-miss: true - name: Add LLVM to PATH @@ -31,6 +104,7 @@ jobs: - uses: taiki-e/upload-rust-binary-action@v1 with: bin: bpf-linker + target: ${{ matrix.target }} features: llvm-sys/force-static env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/tests.rs b/tests/tests.rs index 813c4dcf..cc8ee856 100644 --- a/tests/tests.rs +++ b/tests/tests.rs @@ -1,11 +1,15 @@ use std::{ env, ffi::{OsStr, OsString}, - fs, + fs::{self, File}, + io::Write, + os::unix::fs::PermissionsExt, path::{Path, PathBuf}, process::Command, }; +use regex::Regex; + fn find_binary(binary_re_str: &str) -> PathBuf { let binary_re = regex::Regex::new(binary_re_str).unwrap(); let mut binary = which::which_re(binary_re).expect(binary_re_str); @@ -20,7 +24,69 @@ fn run_mode( sysroot: Option<&Path>, cfg: Option, ) { - let mut target_rustcflags = format!("-C linker={}", env!("CARGO_BIN_EXE_bpf-linker")); + // Check if we are running tests for a cross environment. We do it by + // checking the path of our executable (the first argument) and trying to + // extract the target triple from it. Then we check whether the + // architecture from the triple refers to a foreign architecture. + // running tests for a non-host target. + let pattern = Regex::new(r"/target/(.*?)/debug/deps").unwrap(); + let cmd = env::args().next().unwrap(); + let target_triple = pattern + .captures(&cmd) + .and_then(|caps| caps.get(1)) + .map(|r#match| r#match.as_str().to_string()); + let arch = String::from_utf8_lossy(&Command::new("uname").arg("-m").output().unwrap().stdout) + .trim() + .to_string(); + + let bpf_linker_exe = match target_triple { + Some(target_triple) => { + if target_triple.starts_with(&arch) { + env!("CARGO_BIN_EXE_bpf-linker") + } else { + // For foreign targets, we need to point to a correct bpf-linker binary and + // wrap it in QEMU. + let bpf_linker_exe = format!( + "{}/target/{target_triple}/debug/bpf-linker", + env!("CARGO_MANIFEST_DIR") + ); + let qemu_exe = match target_triple.as_str() { + "aarch64-unknown-linux-gnu" | "aarch64-unknown-linux-musl" => "qemu-aarch64", + "riscv64gc-unknown-linux-gnu" | "riscv64gc-unknown-linux-musl" => { + "qemu-riscv64" + } + "x86_64-unknown-linux-gnu" | "x86_64-unknown-linux-musl" => "qemu-x86_64", + _ => { + panic!("Unsupported target triple: {target_triple}") + } + }; + + // Create a wrapper script which runs bpf-linker with qemu. + // + // Unfortunately, passing + // `qemu-aarch64 ./target/aarch64-uknown-linux-musl/debug/bpf-linker` + // (or any multiple arguments) as `linker` in RUSTFLAGS doesn't work. + let script_path = Path::new("/tmp/qemu_bpf_linker_wrapper.sh"); + let script_content = format!( + r#"#!/bin/bash + +{qemu_exe} "{bpf_linker_exe}" "$@" +"# + ); + let mut file = File::create(script_path).unwrap(); + file.write_all(script_content.as_bytes()).unwrap(); + let metadata = file.metadata().unwrap(); + let mut permissions = metadata.permissions(); + permissions.set_mode(0o755); + file.set_permissions(permissions).unwrap(); + + script_path.to_str().unwrap() + } + } + None => env!("CARGO_BIN_EXE_bpf-linker"), + }; + + let mut target_rustcflags = format!("-C linker={bpf_linker_exe}"); if let Some(sysroot) = sysroot { let sysroot = sysroot.to_str().unwrap(); target_rustcflags += &format!(" --sysroot {sysroot}"); @@ -141,6 +207,9 @@ fn compile_test() { Some(&directory), None::, ); + // TODO(vadorovsky): Make our own BTF dump tooling as part of aya-tool and + // use it here to make BTF tests possible on macOS. + #[cfg(not(target_os = "macos"))] run_mode( target, "assembly",