chore: Build bpf-linker and LLVM for multiple targets

Apart from the x86_64 Linux build, cross compile also to Linux arm64, macOS x86_64 and macoOS arm64.
aya-rs · Oct 12, 2023 · 956d761 · 956d761
1 parent 20e2bfd
commit 956d761
Show file tree

Hide file tree

Showing 3 changed files with 108 additions and 24 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -48,7 +48,6 @@ jobs:
         run: cargo fmt --all -- --check
 
   build:
-    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:
@@ -59,7 +58,21 @@ jobs:
         llvm:
           - 17
           - source
-    name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }}
+        include:
+          - target: aarch64-unknown-linux-musl
+            llvm-target: aarch64-linux-gnu
+            os: ubuntu-latest
+          - target: aarch64-apple-darwin
+            llvm-target: aarch64-apple-darwin
+            os: macos-latest
+          - target: x86_64-unknown-linux-musl
+            llvm-target: x86_64-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-apple-darwin
+            llvm-target: x86_64-apple-darwin
+            os: macos-latest
+    name: rustc=${{ matrix.rust }} llvm=${{ matrix.llvm }} target=${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
     needs: llvm
 
     env:
@@ -85,14 +98,19 @@ jobs:
 
       - uses: Swatinem/rust-cache@v2
 
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        if: startsWith(matrix.os, 'ubuntu')
+        with:
+          target: ${{ matrix.target }}
+
       - name: Check (default features, no system LLVM)
         run: cargo check
 
       - name: Build (default features, no system LLVM)
         run: cargo build
 
-      - name: Install dependencies
-        if: matrix.rust == 'nightly'
+      - name: Install dependencies (Linux)
+        if: matrix.rust == 'nightly' && startsWith(matrix.os, 'ubuntu')
         # ubuntu-22.04 comes with clang 14[0] which doesn't include support for signed and 64bit
         # enum values which was added in clang 15[1].
         #
@@ -110,8 +128,8 @@ jobs:
           sudo apt update
           sudo apt -y install clang gcc-multilib
 
-      - name: Install LLVM
-        if: matrix.llvm != 'source'
+      - name: Install LLVM (Linux)
+        if: matrix.llvm != 'source' && startsWith(matrix.os, 'ubuntu')
         run: |
           set -euxo pipefail
           wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc
@@ -121,12 +139,22 @@ jobs:
           sudo apt -y install llvm-${{ matrix.llvm }}-dev
           echo /usr/lib/llvm-${{ matrix.llvm }}/bin >> $GITHUB_PATH
 
+      - name: Install LLVM (macOS)
+        if: matrix.llvm != 'source' && startsWith(matrix.os, 'macos')
+        run: |
+          set -euxo pipefail
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          echo "/usr/local/bin/brew" >> $GITHUB_PATH
+          brew install llvm
+          echo "$(brew --prefix llvm)/bin" >> $GITHUB_PATH
+          echo "LLVM_SYS_170_PREFIX=$(brew --prefix llvm)" >> $GITHUB_ENV
+
       - name: Restore LLVM
         if: matrix.llvm == 'source'
         uses: actions/cache/restore@v3
         with:
           path: llvm-install
-          key: ${{ needs.llvm.outputs.cache-key }}
+          key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.llvm-target)] }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH && LD_LIBRARY_PATH
@@ -151,6 +179,10 @@ jobs:
       - name: Check
         run: cargo hack check --feature-powerset --features llvm-sys/force-dynamic
 
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3
+
       - name: Build
         run: cargo hack build --feature-powerset --features llvm-sys/force-dynamic
 
@@ -159,19 +191,19 @@ jobs:
         run: cargo hack test --feature-powerset --features llvm-sys/force-dynamic
 
       - uses: actions/checkout@v4
-        if: matrix.rust == 'nightly'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly'
         with:
           repository: aya-rs/aya
           path: aya
           submodules: recursive
 
       - name: Install
-        if: matrix.rust == 'nightly'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly'
         run: cargo install --path . --no-default-features --features llvm-sys/force-dynamic
 
       # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
       - name: Download debian kernels
-        if: matrix.rust == 'nightly' && runner.arch == 'ARM64'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' && runner.arch == 'ARM64'
         working-directory: aya
         run: |
           set -euxo pipefail
@@ -182,7 +214,7 @@ jobs:
 
       # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
       - name: Download debian kernels
-        if: matrix.rust == 'nightly' && runner.arch == 'X64'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly' && runner.arch == 'X64'
         working-directory: aya
         run: |
           set -euxo pipefail
@@ -193,15 +225,15 @@ jobs:
 
       # TODO: Remove this and run the integration tests on the local machine when they pass on 5.15.
       - name: Extract debian kernels
-        if: matrix.rust == 'nightly'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly'
         working-directory: aya
         run: |
           set -euxo pipefail
           find test/.tmp -name '*.deb' -print0 | xargs -t -0 -I {} \
             sh -c "dpkg --fsys-tarfile {} | tar -C test/.tmp --wildcards --extract '*vmlinuz*' --file -"
 
       - name: Run aya integration tests
-        if: matrix.rust == 'nightly'
+        if: startsWith(matrix.os, 'ubuntu') && matrix.rust == 'nightly'
         working-directory: aya
         run: |
           set -euxo pipefail

diff --git a/.github/workflows/llvm.yml b/.github/workflows/llvm.yml
@@ -2,13 +2,32 @@ name: LLVM
 
 on:
   workflow_call:
+    # NOTE(vadorovsky): Unfortunately, Github Actions don't support matrix
+    # strategy in `on` section. So we have to specify outputs by hand.
     outputs:
-      cache-key:
-        value: ${{ jobs.llvm.outputs.cache-key }}
+      cache-key-aarch64-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-linux-gnu }}
+      cache-key-aarch64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-aarch64-apple-darwin }}
+      cache-key-x86_64-linux-gnu:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-linux-gnu }}
+      cache-key-x86_64-apple-darwin:
+        value: ${{ jobs.llvm.outputs.cache-key-x86_64-apple-darwin }}
 
 jobs:
   llvm:
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-linux-gnu
+            os: ubuntu-latest
+          - target: aarch64-apple-darwin
+            os: macos-latest
+          - target: x86_64-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-apple-darwin
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
     name: llvm
     outputs:
       cache-key: ${{ steps.cache-key.outputs.cache-key }}
@@ -20,18 +39,18 @@ jobs:
           echo "sha=$value" >> "$GITHUB_OUTPUT"
 
       - id: cache-key
-        run: echo "cache-key=llvm-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
+        run: echo "cache-key-${{ matrix.target }}=llvm-${{ matrix.target }}-${{ steps.ls-remote.outputs.sha }}-1" >> "$GITHUB_OUTPUT"
 
       - name: Cache
         id: cache-llvm
         uses: actions/cache@v3
         with:
           path: llvm-install
-          key: ${{ steps.cache-key.outputs.cache-key }}
+          key: ${{ steps.cache-key.outputs[format('cache-key-{0}', matrix.target)] }}
           lookup-only: true
 
-      - name: Install Tools
-        if: steps.cache-llvm.outputs.cache-hit != 'true'
+      - name: Install Tools (Linux)
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && startsWith(matrix.os, 'ubuntu')
         run: |
           set -euxo pipefail
           wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | \
@@ -43,6 +62,14 @@ jobs:
           sudo apt update
           sudo apt -y install cmake ninja-build clang lld
 
+      - name: Install Tools (macOS)
+        if: steps.cache-llvm.outputs.cache-hit != 'true' && startsWith(matrix.os, 'macos-latest')
+        run: |
+          set -euxo pipefail
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
+          echo "/usr/local/bin/brew" >> $GITHUB_PATH
+          brew install cmake ninja
+
       - name: Checkout LLVM Source
         if: steps.cache-llvm.outputs.cache-hit != 'true'
         uses: actions/checkout@v4
@@ -55,6 +82,10 @@ jobs:
         if: steps.cache-llvm.outputs.cache-hit != 'true'
         run: |
           set -euxo pipefail
+          LLVM_USE_LINKER_OPTION=""
+          if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then
+            LLVM_USE_LINKER_OPTION="-DLLVM_USE_LINKER=lld"
+          fi
           cmake \
             -S llvm-project/llvm \
             -B llvm-build \
@@ -64,13 +95,15 @@ jobs:
             -DCMAKE_CXX_COMPILER=clang++ \
             -DCMAKE_INSTALL_PREFIX="${{ github.workspace }}/llvm-install" \
             -DLLVM_BUILD_LLVM_DYLIB=ON \
+            -DLLVM_DEFAULT_TARGET_TRIPLE=${{ matrix.target }} \
             -DLLVM_ENABLE_ASSERTIONS=ON \
             -DLLVM_ENABLE_PROJECTS= \
             -DLLVM_ENABLE_RUNTIMES= \
+            -DLLVM_HOST_TRIPLE=${{ matrix.target }} \
             -DLLVM_INSTALL_UTILS=ON \
             -DLLVM_LINK_LLVM_DYLIB=ON \
             -DLLVM_TARGETS_TO_BUILD=BPF \
-            -DLLVM_USE_LINKER=lld
+            $LLVM_USE_LINKER_OPTION
 
       - name: Install LLVM
         if: steps.cache-llvm.outputs.cache-hit != 'true'

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -9,15 +9,29 @@ jobs:
     uses: ./.github/workflows/llvm.yml
 
   upload-bins:
-    # TODO: Build for macos someday.
-    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        include:
+          - target: aarch64-unknown-linux-musl
+            llvm-target: aarch64-linux-gnu
+            os: ubuntu-latest
+          - target: aarch64-apple-darwin
+            llvm-target: aarch64-apple-darwin
+            os: macos-latest
+          - target: x86_64-unknown-linux-musl
+            llvm-target: x86_64-linux-gnu
+            os: ubuntu-latest
+          - target: x86_64-apple-darwin
+            llvm-target: x86_64-apple-darwin
+            os: macos-latest
+    runs-on: ${{ matrix.os }}
     needs: llvm
     steps:
       - name: Restore LLVM
         uses: actions/cache/restore@v3
         with:
           path: llvm-install
-          key: ${{ needs.llvm.outputs.cache-key }}
+          key: ${{ needs.llvm.outputs[format('cache-key-{0}', matrix.llvm-target)] }}
           fail-on-cache-miss: true
 
       - name: Add LLVM to PATH
@@ -28,9 +42,14 @@ jobs:
       - uses: actions/checkout@v4
       - uses: Swatinem/rust-cache@v2
 
+      - uses: taiki-e/setup-cross-toolchain-action@v1
+        if: startsWith(matrix.os, 'ubuntu')
+        with:
+          target: ${{ matrix.target }}
       - uses: taiki-e/upload-rust-binary-action@v1
         with:
           bin: bpf-linker
+          target: ${{ matrix.target }}
           features: llvm-sys/force-static
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}