Cache some of the build using Docker

.dockerignore

@@ -0,0 +1,4 @@
+.github
+.git
+Dockerfile
+README.md

.github/workflows/all.yml

@@ -1,111 +1,27 @@
 name: Build and run everything
+
+# The following should ensure that the workflow only runs a single set of actions
+# for each PR. But it will not apply this to pushes to the main branch.
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 on:
   push:
   pull_request:
   schedule:
     - cron: '0 0 * * 1'
+
 jobs:
-  build-all-alpine:
-    runs-on: ubuntu-latest
-    container:
-      image: alpine:latest
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - name: Install bash
-        run: apk add bash git
-      - name: Silence some git warnings
-        run: |
-          git config --global advice.detachedHead false
-          git config --global init.defaultBranch main
-      - name: Install and build all benchmarks and allocators
-        # dh: glibc-specific
-        # fg: Uses execinfo.h, which is a GNU extension
-        # gd: ?
-        # hd: glibc-specific
-        # lf: crashes redis server
-        # lt: return type 'struct mallinfo' is incomplete
-        # mesh/nomesh: infinite loop?
-        # pa: can't setup depot_tools and goma
-        # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW'
-        # tcg: [...] specifies less restrictive attribute than its target [...]
-        # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-        # rp: rpmalloc/rpmalloc.c:980:9: error: unsafe pointer arithmetic [-Werror,-Wunsafe-buffer-usage]
-        run: ./build-bench-env.sh all no-dh no-hd no-sm no-mesh no-nomesh no-pa no-gd no-fg no-lf no-lt no-tcg no-lp no-rp
-      - name: Run everything.
-        run: |
-          cd out/bench
-          ../../bench.sh alla allt
-  build-all-ubuntu:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - name: Silence some git warnings
-        run: |
-          git config --global advice.detachedHead false
-          git config --global init.defaultBranch main
-      - name: Install and build all benchmarks and allocators
-        # fg: crashes on redis
-        # gd: infinite loop in the redis benchmark
-        # lt: breaks on sh8benchN
-        # lf: crashes redis server
-        # ff: crashes on modern ubuntu: https://github.com/bwickman97/ffmalloc/issues/5
-        # hoard: crashes on rocksdb
-        # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory
-        # sn: /usr/bin/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/chrono:2320:48: error: call to consteval function 'std::chrono::hh_mm_ss::_S_fractional_width' is not a constant expression
-        # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2
-        # tcg: ERROR: Error computing the main repository mapping: protobuf@29.0 depends on rules_fuzzing@0.5.2 with compatibility level 0, but <root> depends on rules_fuzzing@0.5.1 with compatibility level 1 which is different
-        run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-lt no-lf no-hd no-pa no-sn no-lp no-tcg
-      - name: Run everything.
-        run: |
-          cd out/bench
-          ../../bench.sh alla allt
-  build-all-fedora:
-    runs-on: ubuntu-latest
-    container:
-      image: fedora:latest
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - name: Install git
-        run: sudo dnf -y --quiet --nodocs install git
-      - name: Silence some git warnings
-        run: |
-          git config --global advice.detachedHead false
-          git config --global init.defaultBranch main
-      - name: Install and build all benchmarks and allocators
-        # gd: infinite loop in the redis benchmark
-        # mesh/nomesh: error: '__malloc_hook' was not declared in this scope;
-        # mi: error: '__malloc_hook' was not declared in this scope;
-        # rp: mixing declarations and code is incompatible with standards before C99
-        # lf: crashes redis server
-        # fg: crashes redis server
-        # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory
-        # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
-        # tcg: https://github.com/bazelbuild/bazel/issues/19295 bazel5 is required but unavailable
-        run: ./build-bench-env.sh all no-lean no-mi no-mesh no-nomesh no-gd no-rp no-lf no-fg no-pa no-lp no-sh8 no-tcg
-      - name: Run everything.
-        run: |
-          cd out/bench
-          ../../bench.sh alla allt
-  build-all-osx:
-    runs-on: macos-11
-    steps:
-      - name: Check out repository code
-        uses: actions/checkout@v3
-      - name: Silence some git warnings
-        run: |
-          git config --global advice.detachedHead false
-          git config --global init.defaultBranch main
-      - name: Install and build all benchmarks and allocators
-        # ff:ffmalloc.c:1140:14: error: implicit declaration of function 'sched_getcpu' is invalid in C99 [-Werror,-Wimplicit-function-declaration] 
-        # fg: unknown type name 'pthread_spinlock_t'; did you mean 'pthread_rwlock_t'?
-        # gd: so many errors
-        # pa: ninja: error: '../../buildtools/third_party/libc++/trunk/src/utility.cpp', needed by 'obj/buildtools/third_party/libc++/libc++/utility.o', missing and no known rule to make it
-        # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2
-        run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-pa no-lp
-      - name: Run everything.
-        run: |
-          cd out/bench
-          ../../bench.sh alla allt
+  build-and-run:
+    strategy:
+      matrix:
+        platform: [ubuntu, alpine, fedora]
+        exclude:
+          # Fedora is not currently building lean.
+          - platform: fedora
+      fail-fast: false
+    uses: ./.github/workflows/platform.yml
+    name: ${{ matrix.platform }}
+    with:
+      platform: ${{ matrix.platform }}

.github/workflows/platform.yml

@@ -0,0 +1,100 @@
+name: Build and run everything
+on:
+  workflow_call:
+    inputs:
+      platform:
+        description: 'Base container image to build on'
+        required: true
+        type: string
+
+jobs:
+  build-base-with-docker:
+    runs-on: ubuntu-latest
+    name: Base Container
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build benchmarks in Docker
+        id: docker_build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile
+          push: false
+          tags: bench
+          target: bench-env
+          build-args: |
+            platform=${{ inputs.platform }}
+          cache-from: type=gha,scope=buildkit_${{ inputs.platform }}
+          cache-to: type=gha,mode=max,scope=buildkit_${{ inputs.platform }}
+
+  build-and-benchmark-allocator:
+    runs-on: ubuntu-latest
+    needs: build-base-with-docker
+    strategy:
+      matrix:
+        allocator: [ff, fg, gd, hd, hm, iso, je, lf, lp, lt, mesh, mng, mi, mi2, nomesh, pa, rp, scudo, sm, sn, sg, tbb, tc, yal, tcg]
+        testsuite: [allt]
+        # Used to provide a platform-specific predicate
+        predicate: [true, false]
+        exclude:
+          # Always exclude build when predicate is false.
+          - predicate: false
+
+          # Not currently working on any platform
+          - allocator: pa
+
+          # Alpine Specific disables.
+          # dh: glibc-specific
+          - allocator: dh
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # fg: Uses execinfo.h, which is a GNU extension
+          - allocator: fg
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # gd: ?
+          - allocator: gd
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # hd: glibc-specific
+          - allocator: hd
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration]
+          - allocator: lp
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # lt: return type 'struct mallinfo' is incomplete
+          - allocator: lt
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW'
+          - allocator: sm
+            predicate: ${{ inputs.platform == 'alpine' }}
+          # tcg: [...] specifies less restrictive attribute than its target [...]
+          - allocator: tcg
+            predicate: ${{ inputs.platform == 'alpine' }}
+      fail-fast: false
+    name: ${{ matrix.allocator }} - build and benchmark
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Build allocator and run benchmarks in Docker
+        id: docker_build
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: Dockerfile
+          push: false
+          target: benchmark
+          ## If allocators in lt,lf,fg,gd then use a smaller set of tests.
+          ## They were generally failing on lean or redis, and lf was also failing on mstress.
+          build-args: |
+            platform=${{ inputs.platform }}
+            allocator=${{ matrix.allocator }}
+            benchs=${{ (contains('lt,lf,gd', matrix.allocator) && 'cfrac espresso barnes larson-sized mstress rptest gs lua') || ('fg' == matrix.allocator && 'cfrac espresso barnes gs lua') || 'allt' }}
+            repeats=1
+          cache-from: type=gha,scope=buildkit_${{ inputs.platform }}

Dockerfile

@@ -0,0 +1,44 @@
+ARG platform=ubuntu
+
+FROM ubuntu:24.04 AS ubuntu
+
+
+FROM fedora:latest AS fedora
+
+
+FROM alpine:latest AS alpine
+RUN apk add --no-cache bash
+
+
+FROM ${platform} AS bench-env
+
+# Pull mimalloc-bench
+RUN mkdir -p /mimalloc-bench
+COPY . /mimalloc-bench
+
+WORKDIR /mimalloc-bench
+# Install dependencies
+RUN ./build-bench-env.sh packages
+
+# Build benchmarks
+RUN ./build-bench-env.sh bench
+
+RUN ./build-bench-env.sh redis
+
+RUN ./build-bench-env.sh rocksdb
+
+RUN ./build-bench-env.sh lean
+
+FROM bench-env AS benchmark
+
+WORKDIR /mimalloc-bench
+
+ARG allocator=mi
+ARG benchs=cfrac
+ARG repeats=1
+
+RUN ./build-bench-env.sh $allocator
+
+# Run benchmarks
+WORKDIR /mimalloc-bench/out/bench
+RUN ../../bench.sh $allocator $benchs -r=$repeats

build-bench-env.sh

@@ -441,7 +441,7 @@ if test "$setup_packages" = "1"; then
     # no 'apt update' equivalent needed on Fedora
     dnfinstall "gcc-c++ clang lld llvm-devel unzip dos2unix bc gmp-devel wget gawk \
       cmake python3 ruby ninja-build libtool autoconf git patch time sed \
-      ghostscript libatomic which gflags-devel xz readline-devel snappy-devel"
+      ghostscript libatomic libstdc++ which gflags-devel xz readline-devel snappy-devel"
     # bazel5 is broken on the copr: https://github.com/bazelbuild/bazel/issues/19295
     #dnfinstallbazel
   elif grep -q -e 'ID=debian' -e 'ID=ubuntu' /etc/os-release 2>/dev/null; then
@@ -656,7 +656,8 @@ if test "$setup_sn" = "1"; then
   else
     mkdir -p release
     cd release
-    env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release
+    # CXX11_DESTRUCTORS is needed as broken on Alpine without, should be fixed in the future upstrem.
+    env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_CLEANUP=CXX11_DESTRUCTORS
     cd ..
   fi
   cd release