Merge pull request #119 from ashvardanian/main-dev

Make: Cross-compilation
ashvardanian · Mar 17, 2024 · adcba16 · adcba16
2 parents c9928c2 + ca3a410
commit adcba16
Show file tree

Hide file tree

Showing 8 changed files with 238 additions and 139 deletions.
diff --git a/.github/workflows/Dockerfile.libs b/.github/workflows/Dockerfile.libs
diff --git a/.github/workflows/build_tools.sh b/.github/workflows/build_tools.sh
@@ -27,11 +27,11 @@ esac
 # Set build type
 case "$BUILD_TYPE" in
     "Debug")
-        BUILD_DIR="./build_debug"
+        BUILD_DIR="build_debug"
         BUILD_FLAGS="-DCMAKE_BUILD_TYPE=Debug"
         ;;
     "Release")
-        BUILD_DIR="./build_release"
+        BUILD_DIR="build_release"
         BUILD_FLAGS="-DCMAKE_BUILD_TYPE=RelWithDebInfo"
         ;;
     *)

diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -31,11 +31,13 @@ jobs:
         # C/C++
         # If the compilation fails, we want to log the compilation commands in addition to
         # the standard output.
-      - name: Build C/C++
+      - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y cmake build-essential libjemalloc-dev libomp-dev gcc-12 g++-12
 
+      - name: Build C/C++
+        run: |
           cmake -B build_artifacts \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
@@ -56,14 +58,14 @@ jobs:
             exit 1
           }
       - name: Test C++
-        run: ./build_artifacts/stringzilla_test_cpp20
+        run: build_artifacts/stringzilla_test_cpp20
       - name: Test on Real World Data
         run: |
-          ./build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
-          ./build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
-          ./build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          ./build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
-          ./build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
+          build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
+          build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
+          build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
+          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
         # Don't overload GitHub with our benchmarks.
@@ -104,20 +106,30 @@ jobs:
     env:
       CC: clang-16
       CXX: clang++-16
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            target: x86_64-linux-gnu
+          - arch: arm64
+            target: aarch64-linux-gnu
 
     steps:
       - uses: actions/checkout@v4
 
         # C/C++
         # Clang 16 isn't available from default repos on Ubuntu 22.04, so we have to install it manually
-      - name: Build C/C++
+      - name: Install dependencies
         run: |
           sudo apt update
           sudo apt install -y cmake build-essential libjemalloc-dev
           wget https://apt.llvm.org/llvm.sh
           chmod +x llvm.sh
           sudo ./llvm.sh 16
 
+      - name: Build C/C++
+        run: |
           cmake -B build_artifacts \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
@@ -138,14 +150,14 @@ jobs:
             exit 1
           }
       - name: Test C++
-        run: ./build_artifacts/stringzilla_test_cpp20
+        run: build_artifacts/stringzilla_test_cpp20
       - name: Test on Real World Data
         run: |
-          ./build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
-          ./build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
-          ./build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          ./build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
-          ./build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
+          build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
+          build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
+          build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
+          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
         # Don't overload GitHub with our benchmarks.
@@ -186,14 +198,65 @@ jobs:
   # Temporary workaround to run Swift tests on Linux
   # Based on: https://github.com/swift-actions/setup-swift/issues/591#issuecomment-1685710678
   test_ubuntu_swift:
-    name: Ubuntu (Swift)
+    name: Swift on Linux
     runs-on: ubuntu-22.04
     container: swift:5.9
     steps:
       - uses: actions/checkout@v4
       - name: Test Swift
         run: swift test
 
+  test_ubuntu_cross_compilation:
+    name: Cross Compilation
+    runs-on: ubuntu-22.04
+    env:
+      CC: clang
+      CXX: clang++
+      AR: llvm-ar
+      NM: llvm-nm
+      RANLIB: llvm-ranlib
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - arch: amd64
+            target: x86_64-linux-gnu
+          - arch: arm64
+            target: aarch64-linux-gnu
+
+    steps:
+      - uses: actions/checkout@v4
+
+        # C/C++
+        # We need to install the cross-compilation toolchain for ARM64 and ARMHF
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y clang lld make crossbuild-essential-arm64 crossbuild-essential-armhf
+
+      - name: Build C/C++
+        run: |
+          cmake -B build_artifacts \
+            -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
+            -DCMAKE_C_COMPILER_TARGET=${{ matrix.target }} \
+            -DCMAKE_CXX_COMPILER_TARGET=${{ matrix.target }} \
+            -DCMAKE_SYSTEM_NAME=Linux \
+            -DCMAKE_SYSTEM_PROCESSOR=${{ matrix.arch }} \
+            -DSTRINGZILLA_BUILD_SHARED=1 \
+            -DSTRINGZILLA_BUILD_BENCHMARK=1 \
+            -DSTRINGZILLA_BUILD_TEST=1
+
+          cmake --build build_artifacts --config RelWithDebInfo
+
+        # We can't run the produced builds, but we can make sure they exist
+      - name: Test artifacts presense
+        run: |
+          test -f build_artifacts/stringzilla_test_cpp20
+          test -f build_artifacts/libstringzilla.so
+          test -f build_artifacts/libstringzillite.so
+
   test_macos:
     name: MacOS
     runs-on: macos-12
@@ -202,25 +265,27 @@ jobs:
       - uses: actions/checkout@v4
 
         # C/C++
-      - name: Build C/C++
+      - name: Install dependencies
         run: |
           brew update
           brew install cmake
+      - name: Build C/C++
+        run: |
           cmake -B build_artifacts \
             -DCMAKE_BUILD_TYPE=RelWithDebInfo \
             -DCMAKE_EXPORT_COMPILE_COMMANDS=1 \
             -DSTRINGZILLA_BUILD_BENCHMARK=1 \
             -DSTRINGZILLA_BUILD_TEST=1
           cmake --build build_artifacts --config RelWithDebInfo
       - name: Test C++
-        run: ./build_artifacts/stringzilla_test_cpp17
+        run: build_artifacts/stringzilla_test_cpp17
       - name: Test on Real World Data
         run: |
-          ./build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
-          ./build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
-          ./build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
-          ./build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
-          ./build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
+          build_artifacts/stringzilla_bench_search ${DATASET_PATH}     # for substring search
+          build_artifacts/stringzilla_bench_token ${DATASET_PATH}      # for hashing, equality comparisons, etc.
+          build_artifacts/stringzilla_bench_similarity ${DATASET_PATH} # for edit distances and alignment scores
+          build_artifacts/stringzilla_bench_sort ${DATASET_PATH}       # for sorting arrays of strings
+          build_artifacts/stringzilla_bench_container ${DATASET_PATH}  # for STL containers with string keys
         env:
           DATASET_PATH: ./README.md
         # Don't overload GitHub with our benchmarks.
@@ -272,7 +337,7 @@ jobs:
             -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ^
             -DSTRINGZILLA_BUILD_BENCHMARK=1 ^
             -DSTRINGZILLA_BUILD_TEST=1
-          
+
           cmake --build build_artifacts --config RelWithDebInfo > build_artifacts/logs.txt 2>&1 || (
             echo "Compilation failed. Here are the logs:"
             type build_artifacts\logs.txt
@@ -297,7 +362,7 @@ jobs:
         # Don't overload GitHub with our benchmarks.
         # The results in such an unstable environment will be meaningless anyway.
         if: 0
-        
+
         # Python
       - name: Set up Python ${{ env.PYTHON_VERSION }}
         uses: actions/setup-python@v5
@@ -331,7 +396,7 @@ jobs:
             -DSTRINGZILLA_BUILD_TEST=1
           cmake --build build_artifacts --config RelWithDebInfo
       - name: Test C++
-        run: ./build_artifacts/stringzilla_test_cpp20
+        run: build_artifacts/stringzilla_test_cpp20
 
         # Python
       - name: Build Python