Build wheels and precommits with LuaJit and for MacOS/ARM64.

Affects [bazelrc, precommit, make_wheel, luajit.BUILD]. Building for ARM64 on the x86_64 GitHub action runners uses cross-compilation. We add a new configurable setting to allow the genrules that are executed on the host to be aware of the intended target architecture, and we set this setting's value via a command line flag subsumed by the "macos_arm64" config in our .bazelrc. Since the build results are not runnable on the GitHub x86_64 runners, we don't run the unit tests in those configurations.
google-deepmind · May 31, 2023 · ee2f5b5 · ee2f5b5
1 parent 07a2b20
commit ee2f5b5
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 9 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -2,6 +2,7 @@ build --cxxopt=-std=c++17
 build --cxxopt=-Wno-sign-compare --cxxopt=-Wno-attributes
 build --enable_platform_specific_config
 build --flag_alias=lua_version=//dmlab2d/lib/lua:lua_version
+build --flag_alias=macos_target_arch=@luajit_archive//:target_arch
 
 build:libc++ --repo_env=CXXFLAGS=-stdlib=libc++
 build:libc++ --repo_env=LDFLAGS=-stdlib=libc++
@@ -15,3 +16,7 @@ build:luajit --lua_version=luajit
 build:macos --crosstool_top=@local_config_apple_cc//:toolchain
 build:macos --host_crosstool_top=@local_config_apple_cc//:toolchain
 build:macos --apple_crosstool_top=@local_config_apple_cc//:toolchain
+
+build:macos_arm64 --config=macos
+build:macos_arm64 --cpu=darwin_arm64
+build:macos_arm64 --macos_target_arch=arm64
diff --git a/.github/workflows/make_wheel.yml b/.github/workflows/make_wheel.yml
@@ -42,9 +42,13 @@ jobs:
       matrix:
         cfg:
           - { name: 'Linux LLVM+libstdc++', os: 'ubuntu-22.04', platform: 'manylinux_2_35_x86_64', cc: clang, cxx: clang++, config: --linkopt=-fuse-ld=lld }
-          - { name: 'MacOS 11 LLVM+libc++', os: 'macos-11', platform: 'macosx_11_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ }
-          - { name: 'MacOS 12 LLVM+libc++', os: 'macos-12', platform: 'macosx_12_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ }
-          - { name: 'MacOS 13 LLVM+libc++', os: 'macos-13', platform: 'macosx_13_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ }
+          - { name: 'MacOS 11 x86_64 LLVM+libc++', os: 'macos-11', platform: 'macosx_11_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 12 x86_64 LLVM+libc++', os: 'macos-12', platform: 'macosx_12_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 13 x86_64 LLVM+libc++', os: 'macos-13', platform: 'macosx_13_0_x86_64', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 12 ARM64 LLVM+libc++', os: 'macos-12', platform: 'macosx_12_0_arm64', cc: clang, cxx: clang++,
+              config: --config=libc++ --config=macos_arm64 --repo_env=PY_PLATFORM_OVERRIDE=macosx_12_0_arm64 }
+          - { name: 'MacOS 13 ARM64 LLVM+libc++', os: 'macos-13', platform: 'macosx_13_0_arm64', cc: clang, cxx: clang++,
+              config: --config=libc++ --config=macos_arm64 --repo_env=PY_PLATFORM_OVERRIDE=macosx_13_0_arm64 }
         py:
           - { version: '3.8', interp: 'cp38', abi: 'cp38' }
           - { version: '3.9', interp: 'cp39', abi: 'cp39' }
@@ -66,7 +70,7 @@ jobs:
         run: pip3 install --upgrade pip packaging
 
       - name: Build for Python ${{ matrix.py.version }}
-        run: bazel --bazelrc=.bazelrc build --compilation_mode=opt --dynamic_mode=off --config=lua5_1 ${{ matrix.cfg.config }} //dmlab2d:dmlab2d_wheel
+        run: bazel --bazelrc=.bazelrc build --compilation_mode=opt --dynamic_mode=off --config=luajit ${{ matrix.cfg.config }} //dmlab2d:dmlab2d_wheel
 
       - name: Upload Release Asset
         uses: actions/upload-release-asset@v1

diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
@@ -29,32 +29,48 @@ jobs:
           - { name: 'Linux GCC', os: 'ubuntu-22.04', cc: gcc-12, cxx: g++-12, config: }
           - { name: 'Linux LLVM+libstdc++', os: 'ubuntu-22.04', cc: clang, cxx: clang++, config: --linkopt=-fuse-ld=lld }
           - { name: 'Linux LLVM+libc++', os: 'ubuntu-22.04', cc: clang, cxx: clang++, config: --config=libc++ --linkopt=-fuse-ld=lld }
-          - { name: 'MacOS 12 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
-          - { name: 'MacOS 13 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 12 x86_64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 13 x86_64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos }
+          - { name: 'MacOS 12 ARM64 LLVM+libc++', os: 'macos-12', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 }
+          - { name: 'MacOS 13 ARM64 LLVM+libc++', os: 'macos-13', cc: clang, cxx: clang++, config: --config=libc++ --config=macos_arm64 }
 
     env:
       CC:  ${{ matrix.cc }}
       CXX: ${{ matrix.cxx }}
 
     steps:
       - uses: actions/checkout@v3
+
       - name: install infrastructure (Linux)
         if: matrix.os == 'ubuntu-22.04'
         run: |
           sudo apt-get install python3-dev python3-numpy python3-packaging
           sudo ln -s /usr/lib/llvm-10/include/c++/v1 /usr/include/c++/v1
+
       - name: install infrastructure (MacOS)
         if: matrix.os == 'macos-12' || matrix.os == 'macos-13'
         run: sudo -H pip3 install numpy packaging
-      - name: build-and-test-lua5.1
+
+      - name: build-lua5.1
         run: |
           bazel --bazelrc=.bazelrc build --config=lua5_1 ${{ matrix.config }} //...
+      - name: test-lua5.1
+        if: (!contains(matrix.config, 'arm64'))
+        run: |
           bazel --bazelrc=.bazelrc test  --config=lua5_1 ${{ matrix.config }} --test_output=errors //...
-      - name: build-and-test-lua5.2
+
+      - name: build-lua5.2
         run: |
           bazel --bazelrc=.bazelrc build --config=lua5_2 ${{ matrix.config }} //...
+      - name: test-lua5.2
+        if: (!contains(matrix.config, 'arm64'))
+        run: |
           bazel --bazelrc=.bazelrc test  --config=lua5_2 ${{ matrix.config }} --test_output=errors //...
-      - name: build-and-test-luajit
+
+      - name: build-luajit
         run: |
           bazel --bazelrc=.bazelrc build --config=luajit ${{ matrix.config }} //...
+      - name: test-luajit
+        if: (!contains(matrix.config, 'arm64'))
+        run: |
           bazel --bazelrc=.bazelrc test  --config=luajit ${{ matrix.config }} --test_output=errors //...
diff --git a/bazel/luajit.BUILD b/bazel/luajit.BUILD
@@ -1,6 +1,27 @@
 # Description:
 #   Build rule for LuaJit.
 
+load("@bazel_skylib//rules:common_settings.bzl", "string_flag")
+
+string_flag(
+    name = "target_arch",
+    build_setting_default = "x86_64",
+    values = [
+        "arm64",
+        "x86_64",
+    ],
+)
+
+config_setting(
+    name = "target_arch_arm64",
+    flag_values = {"target_arch": "arm64"},
+)
+
+config_setting(
+    name = "target_arch_x86_64",
+    flag_values = {"target_arch": "x86_64"},
+)
+
 UNWINDER_DEFINES = ["LUAJIT_UNWIND_EXTERNAL"]
 
 DEFINES = [
@@ -9,6 +30,7 @@ DEFINES = [
     "LUAJIT_ENABLE_GC64",
 ] + UNWINDER_DEFINES + select(
     {
+        ":target_arch_arm64": ["LUAJIT_TARGET=LUAJIT_ARCH_arm64"],
         "@platforms//cpu:x86_64": ["LUAJIT_TARGET=LUAJIT_ARCH_x64"],
         "@platforms//cpu:aarch64": ["LUAJIT_TARGET=LUAJIT_ARCH_arm64"],
         "@build_bazel_apple_support//configs:darwin_arm64": ["LUAJIT_TARGET=LUAJIT_ARCH_arm64"],
@@ -207,6 +229,7 @@ genrule(
     outs = ["src/host/buildvm_arch.h"],
     cmd = select(
         {
+            ":target_arch_arm64": "touch $(location :src/host/buildvm_arch.h) && $(location :minilua) $(location dynasm/dynasm.lua) -D ENDIAN_LE -D P64 -D JIT -D FFI -D DUALNUM -D FPU -D HFABI -D VER=80 -o $@ $(location :src/vm_arm64.dasc)",
             "@platforms//cpu:x86_64": "touch $(location :src/host/buildvm_arch.h) && $(location :minilua) $(location dynasm/dynasm.lua) -D ENDIAN_LE -D P64 -D JIT -D FFI -D FPU -D HFABI -D VER= -o $@ $(location :src/vm_x64.dasc)",
             "@platforms//cpu:aarch64": "touch $(location :src/host/buildvm_arch.h) && $(location :minilua) $(location dynasm/dynasm.lua) -D ENDIAN_LE -D P64 -D JIT -D FFI -D DUALNUM -D FPU -D HFABI -D VER=80 -o $@ $(location :src/vm_arm64.dasc)",
             "@build_bazel_apple_support//configs:darwin_arm64": "touch $(location :src/host/buildvm_arch.h) && $(location :minilua) $(location dynasm/dynasm.lua) -D ENDIAN_LE -D P64 -D JIT -D FFI -D DUALNUM -D FPU -D HFABI -D VER=80 -o $@ $(location :src/vm_arm64.dasc)",