diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..4717ae1d --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +.github +.git +Dockerfile +README.md \ No newline at end of file diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml index e89ce102..930eee13 100644 --- a/.github/workflows/all.yml +++ b/.github/workflows/all.yml @@ -1,111 +1,27 @@ name: Build and run everything + +# The following should ensure that the workflow only runs a single set of actions +# for each PR. But it will not apply this to pushes to the main branch. +concurrency: + group: ${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + on: push: pull_request: schedule: - cron: '0 0 * * 1' + jobs: - build-all-alpine: - runs-on: ubuntu-latest - container: - image: alpine:latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Install bash - run: apk add bash git - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # dh: glibc-specific - # fg: Uses execinfo.h, which is a GNU extension - # gd: ? - # hd: glibc-specific - # lf: crashes redis server - # lt: return type 'struct mallinfo' is incomplete - # mesh/nomesh: infinite loop? - # pa: can't setup depot_tools and goma - # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW' - # tcg: [...] specifies less restrictive attribute than its target [...] - # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] - # rp: rpmalloc/rpmalloc.c:980:9: error: unsafe pointer arithmetic [-Werror,-Wunsafe-buffer-usage] - run: ./build-bench-env.sh all no-dh no-hd no-sm no-mesh no-nomesh no-pa no-gd no-fg no-lf no-lt no-tcg no-lp no-rp - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-ubuntu: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # fg: crashes on redis - # gd: infinite loop in the redis benchmark - # lt: breaks on sh8benchN - # lf: crashes redis server - # ff: crashes on modern ubuntu: https://github.com/bwickman97/ffmalloc/issues/5 - # hoard: crashes on rocksdb - # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory - # sn: /usr/bin/../lib/gcc/x86_64-linux-gnu/13/../../../../include/c++/13/chrono:2320:48: error: call to consteval function 'std::chrono::hh_mm_ss::_S_fractional_width' is not a constant expression - # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2 - # tcg: ERROR: Error computing the main repository mapping: protobuf@29.0 depends on rules_fuzzing@0.5.2 with compatibility level 0, but depends on rules_fuzzing@0.5.1 with compatibility level 1 which is different - run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-lt no-lf no-hd no-pa no-sn no-lp no-tcg - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-fedora: - runs-on: ubuntu-latest - container: - image: fedora:latest - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Install git - run: sudo dnf -y --quiet --nodocs install git - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # gd: infinite loop in the redis benchmark - # mesh/nomesh: error: '__malloc_hook' was not declared in this scope; - # mi: error: '__malloc_hook' was not declared in this scope; - # rp: mixing declarations and code is incompatible with standards before C99 - # lf: crashes redis server - # fg: crashes redis server - # pa: python3: can't open file '/__w/mimalloc-bench/mimalloc-bench/extern/pa/partition_alloc_builder/tools/rust/update_rust.py': [Errno 2] No such file or directory - # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] - # tcg: https://github.com/bazelbuild/bazel/issues/19295 bazel5 is required but unavailable - run: ./build-bench-env.sh all no-lean no-mi no-mesh no-nomesh no-gd no-rp no-lf no-fg no-pa no-lp no-sh8 no-tcg - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt - build-all-osx: - runs-on: macos-11 - steps: - - name: Check out repository code - uses: actions/checkout@v3 - - name: Silence some git warnings - run: | - git config --global advice.detachedHead false - git config --global init.defaultBranch main - - name: Install and build all benchmarks and allocators - # ff:ffmalloc.c:1140:14: error: implicit declaration of function 'sched_getcpu' is invalid in C99 [-Werror,-Wimplicit-function-declaration] - # fg: unknown type name 'pthread_spinlock_t'; did you mean 'pthread_rwlock_t'? - # gd: so many errors - # pa: ninja: error: '../../buildtools/third_party/libc++/trunk/src/utility.cpp', needed by 'obj/buildtools/third_party/libc++/libc++/utility.o', missing and no known rule to make it - # lp: /home/runner/work/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/hotbit_heap_inlines.h:64:51: error: too few arguments to function call, expected 3, have 2 - run: ./build-bench-env.sh all no-lean no-gd no-ff no-fg no-pa no-lp - - name: Run everything. - run: | - cd out/bench - ../../bench.sh alla allt + build-and-run: + strategy: + matrix: + platform: [ubuntu, alpine, fedora] + exclude: + # Fedora is not currently building lean. + - platform: fedora + fail-fast: false + uses: ./.github/workflows/platform.yml + name: ${{ matrix.platform }} + with: + platform: ${{ matrix.platform }} \ No newline at end of file diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml new file mode 100644 index 00000000..4463bece --- /dev/null +++ b/.github/workflows/platform.yml @@ -0,0 +1,100 @@ +name: Build and run everything +on: + workflow_call: + inputs: + platform: + description: 'Base container image to build on' + required: true + type: string + +jobs: + build-base-with-docker: + runs-on: ubuntu-latest + name: Base Container + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build benchmarks in Docker + id: docker_build + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + push: false + tags: bench + target: bench-env + build-args: | + platform=${{ inputs.platform }} + cache-from: type=gha,scope=buildkit_${{ inputs.platform }} + cache-to: type=gha,mode=max,scope=buildkit_${{ inputs.platform }} + + build-and-benchmark-allocator: + runs-on: ubuntu-latest + needs: build-base-with-docker + strategy: + matrix: + allocator: [ff, fg, gd, hd, hm, iso, je, lf, lp, lt, mesh, mng, mi, mi2, nomesh, pa, rp, scudo, sm, sn, sg, tbb, tc, yal, tcg] + testsuite: [allt] + # Used to provide a platform-specific predicate + predicate: [true, false] + exclude: + # Always exclude build when predicate is false. + - predicate: false + + # Not currently working on any platform + - allocator: pa + + # Alpine Specific disables. + # dh: glibc-specific + - allocator: dh + predicate: ${{ inputs.platform == 'alpine' }} + # fg: Uses execinfo.h, which is a GNU extension + - allocator: fg + predicate: ${{ inputs.platform == 'alpine' }} + # gd: ? + - allocator: gd + predicate: ${{ inputs.platform == 'alpine' }} + # hd: glibc-specific + - allocator: hd + predicate: ${{ inputs.platform == 'alpine' }} + # lp: /__w/mimalloc-bench/mimalloc-bench/extern/lp/Source/bmalloc/libpas/src/libpas/pas_thread_local_cache.c:218:22: error: call to undeclared function 'pthread_getname_np'; ISO C99 and later do not support implicit function declarations [-Wimplicit-function-declaration] + - allocator: lp + predicate: ${{ inputs.platform == 'alpine' }} + # lt: return type 'struct mallinfo' is incomplete + - allocator: lt + predicate: ${{ inputs.platform == 'alpine' }} + # sm: ../src/supermalloc.h:10:31: error: expected initializer before '__THROW' + - allocator: sm + predicate: ${{ inputs.platform == 'alpine' }} + # tcg: [...] specifies less restrictive attribute than its target [...] + - allocator: tcg + predicate: ${{ inputs.platform == 'alpine' }} + fail-fast: false + name: ${{ matrix.allocator }} - build and benchmark + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build allocator and run benchmarks in Docker + id: docker_build + uses: docker/build-push-action@v4 + with: + context: . + file: Dockerfile + push: false + target: benchmark + ## If allocators in lt,lf,fg,gd then use a smaller set of tests. + ## They were generally failing on lean or redis, and lf was also failing on mstress. + build-args: | + platform=${{ inputs.platform }} + allocator=${{ matrix.allocator }} + benchs=${{ (contains('lt,lf,gd', matrix.allocator) && 'cfrac espresso barnes larson-sized mstress rptest gs lua') || ('fg' == matrix.allocator && 'cfrac espresso barnes gs lua') || 'allt' }} + repeats=1 + cache-from: type=gha,scope=buildkit_${{ inputs.platform }} \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..fe54d991 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,44 @@ +ARG platform=ubuntu + +FROM ubuntu:24.04 AS ubuntu + + +FROM fedora:latest AS fedora + + +FROM alpine:latest AS alpine +RUN apk add --no-cache bash + + +FROM ${platform} AS bench-env + +# Pull mimalloc-bench +RUN mkdir -p /mimalloc-bench +COPY . /mimalloc-bench + +WORKDIR /mimalloc-bench +# Install dependencies +RUN ./build-bench-env.sh packages + +# Build benchmarks +RUN ./build-bench-env.sh bench + +RUN ./build-bench-env.sh redis + +RUN ./build-bench-env.sh rocksdb + +RUN ./build-bench-env.sh lean + +FROM bench-env AS benchmark + +WORKDIR /mimalloc-bench + +ARG allocator=mi +ARG benchs=cfrac +ARG repeats=1 + +RUN ./build-bench-env.sh $allocator + +# Run benchmarks +WORKDIR /mimalloc-bench/out/bench +RUN ../../bench.sh $allocator $benchs -r=$repeats \ No newline at end of file diff --git a/build-bench-env.sh b/build-bench-env.sh index 690975b6..4cd9fed1 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -441,7 +441,7 @@ if test "$setup_packages" = "1"; then # no 'apt update' equivalent needed on Fedora dnfinstall "gcc-c++ clang lld llvm-devel unzip dos2unix bc gmp-devel wget gawk \ cmake python3 ruby ninja-build libtool autoconf git patch time sed \ - ghostscript libatomic which gflags-devel xz readline-devel snappy-devel" + ghostscript libatomic libstdc++ which gflags-devel xz readline-devel snappy-devel" # bazel5 is broken on the copr: https://github.com/bazelbuild/bazel/issues/19295 #dnfinstallbazel elif grep -q -e 'ID=debian' -e 'ID=ubuntu' /etc/os-release 2>/dev/null; then @@ -656,7 +656,8 @@ if test "$setup_sn" = "1"; then else mkdir -p release cd release - env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release + # CXX11_DESTRUCTORS is needed as broken on Alpine without, should be fixed in the future upstrem. + env CXX=clang++ cmake -G Ninja .. -DCMAKE_BUILD_TYPE=Release -DSNMALLOC_CLEANUP=CXX11_DESTRUCTORS cd .. fi cd release