From 7423bdeb0e2e2997844537aeb4fac8faafa05358 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 18 Jul 2025 17:26:55 +0200 Subject: [PATCH 01/75] fix rocksdb build on fedora bz backporting an upstream patch --- build-bench-env.sh | 3 +++ patches/rocksdb_build.patch | 13 +++++++++++++ 2 files changed, 16 insertions(+) create mode 100644 patches/rocksdb_build.patch diff --git a/build-bench-env.sh b/build-bench-env.sh index 31f05de8..327badf7 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -763,6 +763,9 @@ if test "$setup_rocksdb" = "1"; then fi cd "rocksdb-$version_rocksdb" + set +e + patch -p1 -N -r- < ../../patches/rocksdb_build.patch > /dev/null + set -e DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 make db_bench -j $procs [ "$CI" ] && find . -name '*.o' -delete popd diff --git a/patches/rocksdb_build.patch b/patches/rocksdb_build.patch new file mode 100644 index 00000000..138708cb --- /dev/null +++ b/patches/rocksdb_build.patch @@ -0,0 +1,13 @@ +# backported from https://github.com/facebook/rocksdb/commit/a4f2d46ab6065eb43ee781b0d57c29787780d935 +diff --git a/db/blob/blob_file_meta.h b/db/blob/blob_file_meta.h +index d7c8a124336..2e47726f8d1 100644 +--- a/db/blob/blob_file_meta.h ++++ b/db/blob/blob_file_meta.h +@@ -6,6 +6,7 @@ + #pragma once + + #include ++#include + #include + #include + #include From 5b538fffa2b69f5e70c7c42a3aec7f31ad4b4d75 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 18 Jul 2025 17:51:41 +0200 Subject: [PATCH 02/75] fix another instance of missing cstdint --- patches/rocksdb_build.patch | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/patches/rocksdb_build.patch b/patches/rocksdb_build.patch index 138708cb..4002f7c8 100644 --- a/patches/rocksdb_build.patch +++ b/patches/rocksdb_build.patch @@ -11,3 +11,18 @@ index d7c8a124336..2e47726f8d1 100644 #include #include #include + +# backported from https://github.com/facebook/rocksdb/commit/e812a3d3b32ac159f2df99bcae4ca55173e790d5 +diff --git a/include/rocksdb/trace_record.h b/include/rocksdb/trace_record.h +index 8f9c3ee2f0f..d0f1532aa3a 100644 +--- a/include/rocksdb/trace_record.h ++++ b/include/rocksdb/trace_record.h +@@ -8,7 +8,7 @@ + #include + #include + #include +- ++#include + #include "rocksdb/rocksdb_namespace.h" + #include "rocksdb/slice.h" + #include "rocksdb/status.h" From 913ab4e0f6c572268b673e171715843985ed1708 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 4 Aug 2025 16:47:49 +0200 Subject: [PATCH 03/75] WIP: rewrite build_bench_env in Make --- Makefile | 140 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ VERSIONS | 27 +++++++++++ notes | 15 ++++++ 3 files changed, 182 insertions(+) create mode 100644 Makefile create mode 100644 VERSIONS create mode 100644 notes diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..2d85af38 --- /dev/null +++ b/Makefile @@ -0,0 +1,140 @@ +CFLAGS+="-march=native" +CXXFLAGS+="-march=native" + +SUDO="sudo" +ifeq ($(shell whoami), 'root') + echo "running as root, avoid doing this if possible." + SUDO="" +endif + +DARWIN="no" +PROCS=$(shell nproc) +EXTSO="so" +SHA256SUM="sha256sum" + +ifeq ($(shell uname), 'Darwin') + DARWIN="yes" + PROCS=$(shell sysctl -n hw.physicalcpu) + EXTSO="dylib" + SHA256SUM="shasum -a 256" +endif + +BENCHMARKS_BIG=lean lua redis rocksdb +ALLOCS = dh ff fg gd hd hm iso je lp lf lt mesh mi mi2 mng nomesh pa rp sc scudo sg sm sn tbb tc tcg yal + +PDFDOC=bench/large.pdf + +.PHONY: all benchmarks benchmarks_all benchmarks_big + +all: $(ALLOCS) benchmarks_all + +benchmarks_all: benchmarks $(BENCHMARKS_BIG) + +benchmarks: bench/CMakeLists.txt bench/shbench/.patched $(PDFDOC) + cmake -B out/bench -S bench + cmake --build out/bench -j $(PROCS) + +PDF_URL=https://raw.githubusercontent.com/geekaaron/Resources/master/resources/Writing_a_Simple_Operating_System--from_Scratch.pdf +#USERAGENT=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0 +$(PDFDOC): + wget --no-verbose -O $(PDFDOC) $(PDF_URL) + +bench/shbench/.patched: bench/shbench/sh6bench.patch bench/shbench/sh6bench.c bench/shbench/sh8bench.patch bench/shbench/SH8BENCH.C + dos2unix $(@D)/*.patch + patch -p1 -o $(@D)/sh6bench-new.c $(@D)/sh6bench.c $(@D)/sh6bench.patch + patch -p1 -o $(@D)/sh8bench-new.c $(@D)/SH8BENCH.C $(@D)/sh8bench.patch + touch $@ + +bench/shbench/sh6bench.c: bench/shbench/bench.zip + cd $(@D) && unzip -o $( build) + gd: modified extern/Makefile to not use sse + lua: moved to big benchmarks because it is built in extern/ + benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) From a886e7bd055a72300dc5eb1e2e0801846d0d63db Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 5 Aug 2025 08:25:11 +0200 Subject: [PATCH 04/75] * WIP: solve a make-dependency problem, put "large PDF" into the right place --- Makefile | 13 ++++++------- notes | 3 ++- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 2d85af38..29cee0a6 100644 --- a/Makefile +++ b/Makefile @@ -20,14 +20,14 @@ ifeq ($(shell uname), 'Darwin') endif BENCHMARKS_BIG=lean lua redis rocksdb -ALLOCS = dh ff fg gd hd hm iso je lp lf lt mesh mi mi2 mng nomesh pa rp sc scudo sg sm sn tbb tc tcg yal +ALLOCS_TRIVIAL = ff fg gd hd iso je lp lf lt mesh mi mi2 mng nomesh pa rp sc sg sm sn tbb tc tcg yal -PDFDOC=bench/large.pdf +PDFDOC=extern/large.pdf -.PHONY: all benchmarks benchmarks_all benchmarks_big - -all: $(ALLOCS) benchmarks_all +.PHONY: all allocs benchmarks benchmarks_all benchmarks_big +all: allocs benchmarks_all +allocs: $(ALLOCS_TRIVIAL) dh hm scudo benchmarks_all: benchmarks $(BENCHMARKS_BIG) benchmarks: bench/CMakeLists.txt bench/shbench/.patched $(PDFDOC) @@ -35,7 +35,6 @@ benchmarks: bench/CMakeLists.txt bench/shbench/.patched $(PDFDOC) cmake --build out/bench -j $(PROCS) PDF_URL=https://raw.githubusercontent.com/geekaaron/Resources/master/resources/Writing_a_Simple_Operating_System--from_Scratch.pdf -#USERAGENT=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:95.0) Gecko/20100101 Firefox/95.0 $(PDFDOC): wget --no-verbose -O $(PDFDOC) $(PDF_URL) @@ -84,7 +83,7 @@ lt_ENV=-C gnu.make.lib ######################################################################## # Todo: check for the big benchmarks -$(ALLOCS) $(BENCHMARKS_BIG): %: extern/%/.built +$(ALLOCS_TRIVIAL) $(BENCHMARKS_BIG): %: extern/%/.built extern/%/.built: extern/%/.unpacked make -C $(@D) $($*_ENV) -j$(PROCS) diff --git a/notes b/notes index 27d07020..739b4861 100644 --- a/notes +++ b/notes @@ -1,8 +1,9 @@ HOMEBREW_NO_EMOJI? put archives in extra storage (and somehow map that for CI) versionize archives for that - get verions into the Makefile + get versions into the Makefile re-echo extern/versions.txt +remove all non-standard pattern targets from the groups, do their stuff explicitly port linux and sjm fg: SSE2RNG is not used in the Makefile From f68d88106fa58b55298a99e627830820802e4ca6 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 6 Aug 2025 17:11:11 +0200 Subject: [PATCH 05/75] * expand. fix a few targets, supply more versions --- Makefile | 99 +++++++++++++++++++++++++++++++++++++++++++++++--------- VERSIONS | 5 ++- notes | 29 ++++++++++------- 3 files changed, 106 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 29cee0a6..197a2273 100644 --- a/Makefile +++ b/Makefile @@ -17,18 +17,19 @@ ifeq ($(shell uname), 'Darwin') PROCS=$(shell sysctl -n hw.physicalcpu) EXTSO="dylib" SHA256SUM="shasum -a 256" + export HOMEBREW_NO_EMOJI=1 endif -BENCHMARKS_BIG=lean lua redis rocksdb -ALLOCS_TRIVIAL = ff fg gd hd iso je lp lf lt mesh mi mi2 mng nomesh pa rp sc sg sm sn tbb tc tcg yal - +BENCHMARKS_EXTERN=lean lua redis rocksdb +ALLOCS_TRIVIAL = ff iso je lf mng sg tbb tc +ALLOCS_NONTRIVIAL = dh fg gd hd hm lp lt mesh mi mi2 nomesh rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf .PHONY: all allocs benchmarks benchmarks_all benchmarks_big all: allocs benchmarks_all -allocs: $(ALLOCS_TRIVIAL) dh hm scudo -benchmarks_all: benchmarks $(BENCHMARKS_BIG) +allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) +benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) benchmarks: bench/CMakeLists.txt bench/shbench/.patched $(PDFDOC) cmake -B out/bench -S bench @@ -76,14 +77,17 @@ gd_ENV=ARC4RNG=1 iso_ENV=library lf_ENV=liblite-malloc-shared.so lt_ENV=-C gnu.make.lib - +hd_ENV=-C src +redis_ENV=USE_JEMALLOC=no MALLOC=libc BUILD_TLS=no -C src +rocksdb_ENV=DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 db_bench ######################################################################## # ALLOCS: generic targets for the standard scheme: download, unpack, # # configure (nop for the standard), compile (using make). # ######################################################################## # Todo: check for the big benchmarks -$(ALLOCS_TRIVIAL) $(BENCHMARKS_BIG): %: extern/%/.built +$(ALLOCS_TRIVIAL) $(BENCHMARKS_EXTERN): %: extern/%/.built +$(ALLOCS_NONTRIVIAL): %: extern/%/.built extern/%/.built: extern/%/.unpacked make -C $(@D) $($*_ENV) -j$(PROCS) @@ -91,7 +95,7 @@ extern/%/.built: extern/%/.unpacked extern/%/.unpacked: archives/%.tar.gz mkdir -p $(@D) - tar -x --strip-components=1 -f $< -C $(@D) + tar -x --strip-components=1 --overwrite -f $< -C $(@D) touch $@ .PRECIOUS: archives/%.tar.gz @@ -102,8 +106,7 @@ archives/%.tar.gz: ######################################################################## # ALLOCS: special cases ######################################################################## - -# dh: uses cmake +#dh: uses cmake extern/dh/.configured: extern/dh/.unpacked cd $(@D) && rm -rf ./benchmarks/ ./src/exterminator/ ./src/local/ ./src/replicated/ ./docs cmake -S $(@D)/src -B $(@D)/build @@ -113,7 +116,13 @@ extern/dh/.built: extern/dh/.configured cmake --build $(@D)/build -j $(PROCS) touch $@ -# hml (hm light): built from the same source, differently configured +#hd: fix in Makefile. If later ported into a patch, hd can be reintegrated with ALLOCS_TRIVIAL +extern/hd/.built: extern/hd/.unpacked + sed -i_orig 's/-arch arm64/ /g' $(@D)/src/GNUmakefile + make -C $(@D) $(hd_ENV) -j$(PROCS) + touch $@ + +#hm/hml (hm light): built from the same source, differently configured hm_ENV=CONFIG_NATIVE=true CONFIG_WERROR=false VARIANT=default hml_ENV=CONFIG_NATIVE=true CONFIG_WERROR=false VARIANT=light extern/hm/.built: extern/hm/.configured @@ -121,11 +130,70 @@ extern/hm/.built: extern/hm/.configured make -C $(@D) $(hml_ENV) -j$(PROCS) touch $@ -# TODO: scudo needs only part of the source archive, maybe port partial_checkout to make and overwrite extern/scudo/.unpacked -extern/scudo/.built: extern/scudo/.configured - cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $CXXFLAGS -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$extso *.cpp -pthread -# TODO: lp the same +# mi,mi2: cmake, and 3 different variants +extern/mi/.built extern/mi2/.built: extern/%/.built: extern/%/.unpacked + cmake -S $(@D) -B $(@D)/out/release + cmake --build $(@D)/out/release -j$(PROCS) + cmake -S $(@D) -B $(@D)/out/debug -DMI_CHECK_FULL=ON + cmake --build $(@D)/out/debug -j$(PROCS) + cmake -S $(@D) -B $(@D)/out/secure -DMI_SECURE=ON + cmake --build $(@D)/out/secure -j$(PROCS) + touch $@ + +#rp: uses ninja, one fix in build.ninja +extern/rp/build.ninja: extern/rp/.unpacked + cd $(@D) && python3 configure.py + sed -i 's/-Werror//' $(@D)/build.ninja + +extern/rp/.built: extern/rp/build.ninja + cd $(@D) && ninja + touch $@ + +#sc: gyp -> make +sc_ENV=BUILDTYPE=Release +extern/sc/.built: extern/sc/Makefile + make -C $(@D) $(sc_ENV) -j$(PROCS) + touch $@ + +extern/sc/Makefile: extern/sc/build/gyp/gyp + cd $(@D) && build/gyp/gyp --depth=. scalloc.gyp + +extern/sc/build/gyp/gyp: extern/sc/.unpacked + cd extern/sc && tools/make_deps.sh + +#scudo: native clang, in a sub-directory +extern/scudo/.built: extern/scudo/.unpacked + cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$extso *.cpp -pthread + touch $@ + +#sm: make, but a fix before +extern/sm/.built: extern/sm/.unpacked + rm -rf ./$(@D)/doc ./$(@D)/paper ./$(@D)/short-talk ./$(@D)/talk + sed -i "s/-Werror//" $(@D)/Makefile.include + make -C $(@D)/release -j$(PROCS) ../release/lib/libsupermalloc.so +#sn: cmake+ninja, builds in sn/release +extern/sn/.built: extern/sn/build.ninja + cd $(@D)/release && ninja libsnmallocshim.$(EXTSO) libsnmallocshim-checks.$(EXTSO) + touch $@ + +extern/sn/build.ninja: extern/sn/.unpacked + env CXX=clang++ cmake -S $(@D) -B $(@D)/release -G Ninja -DCMAKE_BUILD_TYPE=Release + +#tcg: bazel +extern/tcg/.built: extern/tcg/.unpacked + cd $(@D) && bazel build -c opt tcmalloc + touch $@ + +#yal: custom shell script +extern/yal/.built: extern/yal/.unpacked + cd $(@D) && ./build.sh -V + touch $@ + +######################################################################## +# benchmarks residing in ./extern # +######################################################################## +# lean: cmake, additional mathlib setup extern/lean/.built: extern/lean/.unpacked mkdir -p $(@D)/out/release env CC=gcc CXX="g++" cmake -S $(@D)/src -B $(@D)/out/release -DCUSTOM_ALLOCATORS=OFF -DLEAN_EXTRA_CXX_FLAGS="-w" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 @@ -137,3 +205,4 @@ extern/lean/.built: extern/lean/.unpacked # lua only needs to be fetched, not more. extern/lua/.built: extern/lua/.unpacked + touch $@ diff --git a/VERSIONS b/VERSIONS index 1b12db5e..dddb49f7 100644 --- a/VERSIONS +++ b/VERSIONS @@ -15,8 +15,11 @@ mesh: master, d45d6de, https://github.com/plasma-umass/mesh mi2: v2.1.2, 43ce4bd7, https://github.com/microsoft/mimalloc mi: v1.8.2, b66e3214, https://github.com/microsoft/mimalloc mng: master, 2ed5881, https://github.com/richfelker/mallocng-draft -pa: main, e6c10be, https://github.com/1c3t3a/partition_alloc_builder.git +pa: main, e6c10be, https://github.com/1c3t3a/partition_alloc_builder +redis: 6.2.7, e6f6709, https://github.com/redis/redis +rocksdb: 8.1.1, 6a43615, https://github.com/facebook/rocksdb rp: 1.4.5, e4393ff, https://github.com/mjansson/rpmalloc +sc: master, 9ef491c, https://github.com/cksystemsgroup/scalloc scudo: main, 30d3bb59, https://github.com/llvm/llvm-project sg: master, 6d7d065, https://github.com/ssrg-vt/SlimGuard sm: master, 709663f, https://github.com/kuszmaul/SuperMalloc diff --git a/notes b/notes index 739b4861..7b36415d 100644 --- a/notes +++ b/notes @@ -1,16 +1,23 @@ -HOMEBREW_NO_EMOJI? -put archives in extra storage (and somehow map that for CI) - versionize archives for that - get versions into the Makefile - re-echo extern/versions.txt -remove all non-standard pattern targets from the groups, do their stuff explicitly - -port linux and sjm -fg: SSE2RNG is not used in the Makefile - +TODO + put archives in extra storage (and somehow map that for CI) + versionize archives for that + port linux and sjm + hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) + scudo needs only part of the source archive, maybe port partial_checkout to make and overwrite extern/scudo/.unpacked + lp needs only part of WebKit (like scudo): port partial_checkout? + pa: re-add + adapt CI-scripts to use Makefile Decisions made: + VERSIONS: demand versions instead of reporting them in the end dh: move build directory (src/build -> build) gd: modified extern/Makefile to not use sse - lua: moved to big benchmarks because it is built in extern/ + lua: moved to extern benchmarks because it is built in extern/ benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) + sm: instead of default target, switched to building supermalloc only + +Problems: + fg: built purely for x86, can't test + lt: same. + sc: same. + sm: same. NO_RTM=1 during compilation helps, but does not solve From e3b94bbbb4b07fdb0021d66ea8cbeabcc6a30213 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 6 Aug 2025 23:38:19 +0200 Subject: [PATCH 06/75] * Makefile: je: add configure step --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index 197a2273..050ecd8e 100644 --- a/Makefile +++ b/Makefile @@ -130,6 +130,14 @@ extern/hm/.built: extern/hm/.configured make -C $(@D) $(hml_ENV) -j$(PROCS) touch $@ +#je: needs configure +extern/je/.built: extern/je/config.status + make -C $(@D) -j$(PROCS) + touch @ + +extern/je/config.status: extern/je/.unpacked + cd $(@D) && ./autogen.sh --enable-doc=no --enable-static=no --disable-stats + # mi,mi2: cmake, and 3 different variants extern/mi/.built extern/mi2/.built: extern/%/.built: extern/%/.unpacked cmake -S $(@D) -B $(@D)/out/release From 38528d8268451fca723ba62f9aaf31cfe7431f4c Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 6 Aug 2025 23:44:41 +0200 Subject: [PATCH 07/75] * Makefile: fix tbb --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index 050ecd8e..6598a877 100644 --- a/Makefile +++ b/Makefile @@ -188,6 +188,15 @@ extern/sn/.built: extern/sn/build.ninja extern/sn/build.ninja: extern/sn/.unpacked env CXX=clang++ cmake -S $(@D) -B $(@D)/release -G Ninja -DCMAKE_BUILD_TYPE=Release +#tbb: cmake to configure +extern/tbb/.built: extern/tbb/.configured + make -C $(@D) -j$(PROCS) + touch @ + +extern/tbb/.configured: extern/tbb/.unpacked + cd $(@D) && cmake -DCMAKE_BUILD_TYPE=Release -DTBB_BUILD=OFF -DTBB_TEST=OFF -DTBB_OUTPUT_DIR_BASE=bench -DCMAKE_POLICY_VERSION_MINIMUM=3.5 . + touch @ + #tcg: bazel extern/tcg/.built: extern/tcg/.unpacked cd $(@D) && bazel build -c opt tcmalloc From 9eb39c244e117689cfeeeb7dc69fd009ef9d2efd Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 7 Aug 2025 15:26:53 +0200 Subject: [PATCH 08/75] * Makefile: finish on ARM64 for now --- .gitignore | 1 + Makefile | 39 +++++++++++++++++++++++++++------------ notes | 20 ++++++++++++-------- 3 files changed, 40 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index f16f8006..f623cc7e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +archives/ extern/ out/ bench/doc/ diff --git a/Makefile b/Makefile index 6598a877..85d98bd4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ endif BENCHMARKS_EXTERN=lean lua redis rocksdb ALLOCS_TRIVIAL = ff iso je lf mng sg tbb tc -ALLOCS_NONTRIVIAL = dh fg gd hd hm lp lt mesh mi mi2 nomesh rp sc scudo sm sn tcg yal +ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp scudo sn tcg yal PDFDOC=extern/large.pdf .PHONY: all allocs benchmarks benchmarks_all benchmarks_big @@ -31,7 +31,7 @@ all: allocs benchmarks_all allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) -benchmarks: bench/CMakeLists.txt bench/shbench/.patched $(PDFDOC) +benchmarks: bench/CMakeLists.txt bench/shbench/sh6bench-new.c bench/shbench/sh8bench-new.c $(PDFDOC) cmake -B out/bench -S bench cmake --build out/bench -j $(PROCS) @@ -39,11 +39,13 @@ PDF_URL=https://raw.githubusercontent.com/geekaaron/Resources/master/resources/W $(PDFDOC): wget --no-verbose -O $(PDFDOC) $(PDF_URL) -bench/shbench/.patched: bench/shbench/sh6bench.patch bench/shbench/sh6bench.c bench/shbench/sh8bench.patch bench/shbench/SH8BENCH.C - dos2unix $(@D)/*.patch - patch -p1 -o $(@D)/sh6bench-new.c $(@D)/sh6bench.c $(@D)/sh6bench.patch - patch -p1 -o $(@D)/sh8bench-new.c $(@D)/SH8BENCH.C $(@D)/sh8bench.patch - touch $@ +bench/shbench/sh6bench-new.c: bench/shbench/sh6bench.patch bench/shbench/sh6bench.c + dos2unix $< + patch -p1 -o $@ $(filter-out $<, $^) $< + +bench/shbench/sh8bench-new.c: bench/shbench/sh8bench.patch bench/shbench/SH8BENCH.C + dos2unix $< + patch -p1 -o $@ $(filter-out $<, $^) $< bench/shbench/sh6bench.c: bench/shbench/bench.zip cd $(@D) && unzip -o $( build) - gd: modified extern/Makefile to not use sse + switched from git cloning to grabbing a snapshot of the reqeuested version from github + dh: move build directory (dh/src/build -> dh/build) lua: moved to extern benchmarks because it is built in extern/ benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) sm: instead of default target, switched to building supermalloc only Problems: - fg: built purely for x86, can't test + fg: built purely for x86, can't test - need to re-add when tested lt: same. - sc: same. - sm: same. NO_RTM=1 during compilation helps, but does not solve + sc: same. re-add + sm: same. NO_RTM=1 during compilation helps, but does not solve; re-add + lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. + mesh, nomesh: bazel currently only supports x86_64; re-add when tested on x86 From 4c78c055a5e92f78dc6a5a0c17bef45a3214a813 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 08:06:06 +0200 Subject: [PATCH 09/75] add a ToDo for tar selective unpacking --- notes | 1 + 1 file changed, 1 insertion(+) diff --git a/notes b/notes index d1d4f0b2..a8c79426 100644 --- a/notes +++ b/notes @@ -9,6 +9,7 @@ TODO gd: arc4random named a function getentropy which chalshes with a libc functions lean: test bench.sh with smaller make-target: lean adapt CI-scripts to use Makefile + we can give tar the list of files we want to extract or not extract Decisions made: VERSIONS: demand versions instead of reporting them in the end From ffbd5b80d9a7fcb96b6793d5d6ad2067082bfee8 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 08:06:36 +0200 Subject: [PATCH 10/75] re-add fg --- Makefile | 3 ++- notes | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 85d98bd4..7d6de375 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ ifeq ($(shell uname), 'Darwin') endif BENCHMARKS_EXTERN=lean lua redis rocksdb -ALLOCS_TRIVIAL = ff iso je lf mng sg tbb tc +ALLOCS_TRIVIAL = ff fg iso je lf mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp scudo sn tcg yal PDFDOC=extern/large.pdf @@ -74,6 +74,7 @@ dependencies: # respective target name. # ######################################################################## +fg_env=SSE2RNG=1 #Todo: only set this if not running on x86 gd_ENV=ARC4RNG=1 iso_ENV=library diff --git a/notes b/notes index a8c79426..9eec84b9 100644 --- a/notes +++ b/notes @@ -20,7 +20,6 @@ Decisions made: sm: instead of default target, switched to building supermalloc only Problems: - fg: built purely for x86, can't test - need to re-add when tested lt: same. sc: same. re-add sm: same. NO_RTM=1 during compilation helps, but does not solve; re-add From ddcbb287a796bc649c3f2d846b9ebde14aefc6c8 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 08:20:46 +0200 Subject: [PATCH 11/75] lt: re-add --- Makefile | 2 +- notes | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 7d6de375..3425be70 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ ifeq ($(shell uname), 'Darwin') endif BENCHMARKS_EXTERN=lean lua redis rocksdb -ALLOCS_TRIVIAL = ff fg iso je lf mng sg tbb tc +ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp scudo sn tcg yal PDFDOC=extern/large.pdf diff --git a/notes b/notes index 9eec84b9..fe178bc1 100644 --- a/notes +++ b/notes @@ -20,7 +20,6 @@ Decisions made: sm: instead of default target, switched to building supermalloc only Problems: - lt: same. sc: same. re-add sm: same. NO_RTM=1 during compilation helps, but does not solve; re-add lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. From f5077d57d6d5397c74b92ebae4fb1c60e0fa4d22 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 08:21:14 +0200 Subject: [PATCH 12/75] sc: re-add --- Makefile | 2 +- notes | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3425be70..71eafbf3 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ endif BENCHMARKS_EXTERN=lean lua redis rocksdb ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc -ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp scudo sn tcg yal +ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sn tcg yal PDFDOC=extern/large.pdf .PHONY: all allocs benchmarks benchmarks_all benchmarks_big diff --git a/notes b/notes index fe178bc1..524cac7e 100644 --- a/notes +++ b/notes @@ -20,7 +20,6 @@ Decisions made: sm: instead of default target, switched to building supermalloc only Problems: - sc: same. re-add sm: same. NO_RTM=1 during compilation helps, but does not solve; re-add lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. mesh, nomesh: bazel currently only supports x86_64; re-add when tested on x86 From 560ac03fb37d6b4ab8bfd215ff207fe7d4d10cf3 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 08:23:08 +0200 Subject: [PATCH 13/75] sm: re-add --- Makefile | 2 +- notes | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 71eafbf3..1356d6e4 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ endif BENCHMARKS_EXTERN=lean lua redis rocksdb ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc -ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sn tcg yal +ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf .PHONY: all allocs benchmarks benchmarks_all benchmarks_big diff --git a/notes b/notes index 524cac7e..724eb7b1 100644 --- a/notes +++ b/notes @@ -20,6 +20,5 @@ Decisions made: sm: instead of default target, switched to building supermalloc only Problems: - sm: same. NO_RTM=1 during compilation helps, but does not solve; re-add lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. mesh, nomesh: bazel currently only supports x86_64; re-add when tested on x86 From 8270e08742e9627e610d7fd1117b4eedccecbb32 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 14:42:18 +0200 Subject: [PATCH 14/75] do not build allocators on aarch64 that do not build on that platform --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 1356d6e4..31ae5270 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,12 @@ ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf +# TODO: Mac seems to report 'arm64' here +ifeq ($(shell uname -m), aarch64) + ALLOCS_TRIVIAL := $(filter-out fg lt, $(ALLOCS_TRIVIAL)) + ALLOCS_NONTRIVIAL := $(filter-out sc sm, $(ALLOCS_NONTRIVIAL)) +endif + .PHONY: all allocs benchmarks benchmarks_all benchmarks_big all: allocs benchmarks_all From c4e754b18e191621a3a4f1c42b0607de390efb46 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 9 Sep 2025 14:42:40 +0200 Subject: [PATCH 15/75] fix quoting to make style --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 31ae5270..49d6c132 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,22 @@ -CFLAGS+="-march=native" -CXXFLAGS+="-march=native" +CFLAGS+=-march=native +CXXFLAGS+=-march=native -SUDO="sudo" -ifeq ($(shell whoami), 'root') +SUDO=sudo +ifeq ($(shell whoami), root) echo "running as root, avoid doing this if possible." - SUDO="" + SUDO= endif -DARWIN="no" +DARWIN=no PROCS=$(shell nproc) -EXTSO="so" +EXTSO=so SHA256SUM="sha256sum" -ifeq ($(shell uname), 'Darwin') - DARWIN="yes" +ifeq ($(shell uname), Darwin) + DARWIN=yes PROCS=$(shell sysctl -n hw.physicalcpu) - EXTSO="dylib" - SHA256SUM="shasum -a 256" + EXTSO=dylib + SHA256SUM=shasum -a 256 export HOMEBREW_NO_EMOJI=1 endif From e44fd39343b0f33b22c413b88fb3d323d7232db4 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 10 Sep 2025 09:40:03 +0200 Subject: [PATCH 16/75] lean: build smaller target --- Makefile | 2 +- notes | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 49d6c132..268dbf08 100644 --- a/Makefile +++ b/Makefile @@ -236,7 +236,7 @@ extern/yal/.built: extern/yal/.unpacked extern/lean/.built: extern/lean/.unpacked mkdir -p $(@D)/out/release env CC=gcc CXX="g++" cmake -S $(@D)/src -B $(@D)/out/release -DCUSTOM_ALLOCATORS=OFF -DLEAN_EXTRA_CXX_FLAGS="-w" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - make -C $(@D)/out/release -j$(PROCS) + make -C $(@D)/out/release -j$(PROCS) lean rm -rf $(@D)/out/release/tests mkdir -p extern/mathlib cp -u $(@D)/leanpkg/leanpkg.toml extern/mathlib diff --git a/notes b/notes index 724eb7b1..b71fffa7 100644 --- a/notes +++ b/notes @@ -7,7 +7,6 @@ TODO pa: re-add gd: detect architecture and set SSE2RNG if on x86, see gd/Makefile gd: arc4random named a function getentropy which chalshes with a libc functions - lean: test bench.sh with smaller make-target: lean adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract From f91afc809563bfa52f3429e15a5fafacf03c3c55 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 10 Sep 2025 11:05:04 +0200 Subject: [PATCH 17/75] document standalone repo for pa to further investigate --- notes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notes b/notes index b71fffa7..b6dd4067 100644 --- a/notes +++ b/notes @@ -4,7 +4,7 @@ TODO port linux hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) scudo needs only part of the source archive, maybe port partial_checkout to make? - pa: re-add + pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator gd: detect architecture and set SSE2RNG if on x86, see gd/Makefile gd: arc4random named a function getentropy which chalshes with a libc functions adapt CI-scripts to use Makefile From c5b86667df18fa8e76a4bc0436bb1f4dbfb56523 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 10 Sep 2025 11:22:57 +0200 Subject: [PATCH 18/75] port building linux to the Makefile --- Makefile | 6 +++++- VERSIONS | 1 + notes | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 268dbf08..45928f4e 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ ifeq ($(shell uname), Darwin) export HOMEBREW_NO_EMOJI=1 endif -BENCHMARKS_EXTERN=lean lua redis rocksdb +BENCHMARKS_EXTERN=lean linux lua redis rocksdb ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf @@ -245,3 +245,7 @@ extern/lean/.built: extern/lean/.unpacked # lua only needs to be fetched, not more. extern/lua/.built: extern/lua/.unpacked touch $@ + +# llinux only needs to be fetched, not more. +extern/linux/.built: extern/linux/.unpacked + touch $@ diff --git a/VERSIONS b/VERSIONS index dddb49f7..dcb8bce5 100644 --- a/VERSIONS +++ b/VERSIONS @@ -8,6 +8,7 @@ iso: 1.2.5, 2670d5f, https://github.com/struct/isoalloc je: 5.3.0, 54eaed1d, https://github.com/jemalloc/jemalloc lean: 21d264a66d53b0a910178ae7d9529cb5886a39b6, 21d264a66, https://github.com/leanprover-community/lean lf: master, 915f51b, https://github.com/Begun/lockfree-malloc +linux: 6.6.105, fe9731e1, https://github.com/torvalds/linux lp: main, 08f06e1d, https://github.com/WebKit/WebKit lt: master, f2d7df0, https://github.com/r-lyeh-archived/ltalloc lua: v5.4.7, 1ab3208a, https://github.com/lua/lua diff --git a/notes b/notes index b6dd4067..176c7fbf 100644 --- a/notes +++ b/notes @@ -1,7 +1,6 @@ TODO put archives in extra storage (and somehow map that for CI) versionize archives for that - port linux hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) scudo needs only part of the source archive, maybe port partial_checkout to make? pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator @@ -17,6 +16,7 @@ Decisions made: lua: moved to extern benchmarks because it is built in extern/ benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) sm: instead of default target, switched to building supermalloc only + linux: updated version to LTS series Problems: lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. From 71b60ce1025bc4566127f4a810bbb2b22761cb2b Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 10 Sep 2025 11:55:40 +0200 Subject: [PATCH 19/75] gd: fix random function selection --- Makefile | 5 ++--- notes | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 45928f4e..442752d8 100644 --- a/Makefile +++ b/Makefile @@ -29,6 +29,8 @@ PDFDOC=extern/large.pdf ifeq ($(shell uname -m), aarch64) ALLOCS_TRIVIAL := $(filter-out fg lt, $(ALLOCS_TRIVIAL)) ALLOCS_NONTRIVIAL := $(filter-out sc sm, $(ALLOCS_NONTRIVIAL)) + # gd uses SSE on x86, but ARC4 on ARM + gd_ENV := ARC4RNG=1 endif .PHONY: all allocs benchmarks benchmarks_all benchmarks_big @@ -81,8 +83,6 @@ dependencies: ######################################################################## fg_env=SSE2RNG=1 -#Todo: only set this if not running on x86 -gd_ENV=ARC4RNG=1 iso_ENV=library lf_ENV=liblite-malloc-shared.so lt_ENV=-C gnu.make.lib @@ -218,7 +218,6 @@ extern/tc/Makefile: extern/tc/configure extern/tc/configure: extern/tc/.unpacked cd $(@D) && ./autogen.sh - #tcg: bazel extern/tcg/.built: extern/tcg/.unpacked cd $(@D) && bazel build -c opt tcmalloc diff --git a/notes b/notes index 176c7fbf..7b8f9400 100644 --- a/notes +++ b/notes @@ -4,7 +4,6 @@ TODO hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) scudo needs only part of the source archive, maybe port partial_checkout to make? pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator - gd: detect architecture and set SSE2RNG if on x86, see gd/Makefile gd: arc4random named a function getentropy which chalshes with a libc functions adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract From f0ae03bebbcd88e8bf6a1201c8d6c1a496236b06 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 10 Sep 2025 15:19:13 +0200 Subject: [PATCH 20/75] add more ToDos --- notes | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/notes b/notes index 7b8f9400..c0ae03b9 100644 --- a/notes +++ b/notes @@ -7,6 +7,9 @@ TODO gd: arc4random named a function getentropy which chalshes with a libc functions adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract + package installation + document in README + move to CI instead of build_bench_env.sh Decisions made: VERSIONS: demand versions instead of reporting them in the end @@ -14,6 +17,7 @@ Decisions made: dh: move build directory (dh/src/build -> dh/build) lua: moved to extern benchmarks because it is built in extern/ benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) + that could be fixed by suppplying a Makefile in bench/, too. sm: instead of default target, switched to building supermalloc only linux: updated version to LTS series From 80f1cf346634299ab3d00417649b8e305ac6ade2 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 12 Sep 2025 09:25:54 +0200 Subject: [PATCH 21/75] re-add mesh --- Makefile | 5 +++-- notes | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 442752d8..ffc250f3 100644 --- a/Makefile +++ b/Makefile @@ -21,13 +21,13 @@ ifeq ($(shell uname), Darwin) endif BENCHMARKS_EXTERN=lean linux lua redis rocksdb -ALLOCS_TRIVIAL = ff fg iso je lf lt mng sg tbb tc +ALLOCS_TRIVIAL = ff fg iso je lf lt mesh mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) - ALLOCS_TRIVIAL := $(filter-out fg lt, $(ALLOCS_TRIVIAL)) + ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) ALLOCS_NONTRIVIAL := $(filter-out sc sm, $(ALLOCS_NONTRIVIAL)) # gd uses SSE on x86, but ARC4 on ARM gd_ENV := ARC4RNG=1 @@ -84,6 +84,7 @@ dependencies: fg_env=SSE2RNG=1 iso_ENV=library +mesh_ENV=build lf_ENV=liblite-malloc-shared.so lt_ENV=-C gnu.make.lib hd_ENV=-C src diff --git a/notes b/notes index c0ae03b9..c1b95b24 100644 --- a/notes +++ b/notes @@ -23,4 +23,4 @@ Decisions made: Problems: lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. - mesh, nomesh: bazel currently only supports x86_64; re-add when tested on x86 + nomesh: bazel currently only supports x86_64; re-add when tested on x86 From c7adfd62a0b683c5f070564ac5ba8674700f3ae5 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 12 Sep 2025 09:50:21 +0200 Subject: [PATCH 22/75] nomesh: re-add --- Makefile | 15 ++++++++++++--- notes | 1 - 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index ffc250f3..065cf087 100644 --- a/Makefile +++ b/Makefile @@ -22,13 +22,13 @@ endif BENCHMARKS_EXTERN=lean linux lua redis rocksdb ALLOCS_TRIVIAL = ff fg iso je lf lt mesh mng sg tbb tc -ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 rp sc scudo sm sn tcg yal +ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 nomesh rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) - ALLOCS_NONTRIVIAL := $(filter-out sc sm, $(ALLOCS_NONTRIVIAL)) + ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) # gd uses SSE on x86, but ARC4 on ARM gd_ENV := ARC4RNG=1 endif @@ -114,7 +114,7 @@ archives/%.tar.gz: wget -O $@ $(shell grep "$*:" VERSIONS | cut -d, -f3)/archive/$(shell grep "$*:" VERSIONS | cut -d, -f2| tr -d ' ').tar.gz ######################################################################## -# ALLOCS: special cases +# ALLOCS: nontrivial cases ######################################################################## #dh: uses cmake extern/dh/.configured: extern/dh/.unpacked @@ -159,6 +159,15 @@ extern/mi/.built extern/mi2/.built: extern/%/.built: extern/%/.unpacked cmake --build $(@D)/out/secure -j$(PROCS) touch $@ +# nomesh: built from mesh's source +nomesh_ENV=DISABLE_MESHING=ON build +extern/nomesh/.built: extern/mesh/.unpacked + cp -r $( Date: Fri, 12 Sep 2025 09:53:14 +0200 Subject: [PATCH 23/75] bench: update dh's location --- bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench.sh b/bench.sh index c03fe2de..5be7d8cb 100755 --- a/bench.sh +++ b/bench.sh @@ -98,7 +98,7 @@ readonly lib_tbb="$localdevdir/tbb/bench_release/libtbbmalloc_proxy$extso" readonly lib_tbb_dir="$(dirname $lib_tbb)" -alloc_lib_add "dh" "$localdevdir/dh/src/build/libdieharder$extso" +alloc_lib_add "dh" "$localdevdir/dh/build/libdieharder$extso" alloc_lib_add "ff" "$localdevdir/ff/libffmallocnpmt$extso" alloc_lib_add "fg" "$localdevdir/fg/libfreeguard$extso" alloc_lib_add "gd" "$localdevdir/gd/libguarder$extso" From f8ed28a9499c127cfb36292be1044d8c44181d2e Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 12 Sep 2025 09:53:54 +0200 Subject: [PATCH 24/75] build_bench_env: document a dependency of gyp for ArchLinux --- build-bench-env.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build-bench-env.sh b/build-bench-env.sh index 4cd9fed1..159174e1 100755 --- a/build-bench-env.sh +++ b/build-bench-env.sh @@ -463,7 +463,9 @@ if test "$setup_packages" = "1"; then brewinstall "dos2unix wget cmake ninja automake libtool gnu-time gmp mpir gnu-sed \ ghostscript bazelisk gflags snappy" elif grep -q 'Arch Linux' /etc/os-release; then - sudo pacman -S dos2unix wget cmake ninja automake libtool time gmp sed ghostscript bazelisk gflags snappy + sudo pacman -S dos2unix wget cmake ninja automake libtool time gmp sed ghostscript \ + bazelisk gflags snappy python-six + # python-six is needed for gyp which is used by sc fi fi From 98dfa6f7c4750c786f01886f83048f5af82f1579 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 12 Sep 2025 10:19:01 +0200 Subject: [PATCH 25/75] notes: small cleanup --- notes | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/notes b/notes index abeda438..28569fef 100644 --- a/notes +++ b/notes @@ -2,8 +2,6 @@ TODO put archives in extra storage (and somehow map that for CI) versionize archives for that hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) - scudo needs only part of the source archive, maybe port partial_checkout to make? - pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator gd: arc4random named a function getentropy which chalshes with a libc functions adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract @@ -13,6 +11,7 @@ TODO Decisions made: VERSIONS: demand versions instead of reporting them in the end + bench.sh needs to be adapted in the way it finds 'installed allocators' switched from git cloning to grabbing a snapshot of the reqeuested version from github dh: move build directory (dh/src/build -> dh/build) lua: moved to extern benchmarks because it is built in extern/ @@ -20,6 +19,11 @@ Decisions made: that could be fixed by suppplying a Makefile in bench/, too. sm: instead of default target, switched to building supermalloc only linux: updated version to LTS series + lean: switched to smaller build target -Problems: - lp needs only part of WebKit (like scudo): port partial_checkout? Also, github does not serve archives on that repo. Re-add when solved. +open Questions: + partial_checkout + lp Also, github does not serve archives on that repo. Re-add when solved. + scudo + pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator + build system uses depot_tools which come with problems, also does not compile even when installed From fed31d04db25d9714af5a4fb5587a56ae8dd4fec Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 15 Sep 2025 16:39:17 +0200 Subject: [PATCH 26/75] gd: fix build on aarch64 --- Makefile | 6 +++++- notes | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 065cf087..433b17ba 100644 --- a/Makefile +++ b/Makefile @@ -29,8 +29,12 @@ PDFDOC=extern/large.pdf ifeq ($(shell uname -m), aarch64) ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) - # gd uses SSE on x86, but ARC4 on ARM + # gd uses SSE on x86, but ARC4 on ARM - and arc4 needs a fix gd_ENV := ARC4RNG=1 +extern/gd/.built: extern/gd/.unpacked + sed -i_orig 's/getentropy(/_getentropy(/g' $(@D)/rng/arc4random.{c,h} + make -C $(@D) $(gd_ENV) -j$(PROCS) + touch $@ endif .PHONY: all allocs benchmarks benchmarks_all benchmarks_big diff --git a/notes b/notes index 28569fef..aca5a767 100644 --- a/notes +++ b/notes @@ -2,7 +2,6 @@ TODO put archives in extra storage (and somehow map that for CI) versionize archives for that hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) - gd: arc4random named a function getentropy which chalshes with a libc functions adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract package installation From 38d8445f550855fa334a790ced1c394d92533d84 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 15 Sep 2025 16:40:53 +0200 Subject: [PATCH 27/75] notes: update --- notes | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/notes b/notes index aca5a767..ca953831 100644 --- a/notes +++ b/notes @@ -1,7 +1,6 @@ TODO put archives in extra storage (and somehow map that for CI) versionize archives for that - hd: Makefile fix (build-bench-env.sh, line 612: port to patch?) adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract package installation @@ -26,3 +25,5 @@ open Questions: scudo pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator build system uses depot_tools which come with problems, also does not compile even when installed + sed patches in Makefile: port to proper patch? + at least gd, hd, rp are patched with sed right now, we could port that to a patch From ed946ba883b94830e7879b2983796c455e265537 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 15 Sep 2025 16:52:45 +0200 Subject: [PATCH 28/75] lean: do not build the tests, also do not delete them afterwards --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 433b17ba..c59b70d5 100644 --- a/Makefile +++ b/Makefile @@ -249,8 +249,7 @@ extern/yal/.built: extern/yal/.unpacked extern/lean/.built: extern/lean/.unpacked mkdir -p $(@D)/out/release env CC=gcc CXX="g++" cmake -S $(@D)/src -B $(@D)/out/release -DCUSTOM_ALLOCATORS=OFF -DLEAN_EXTRA_CXX_FLAGS="-w" -DCMAKE_POLICY_VERSION_MINIMUM=3.5 - make -C $(@D)/out/release -j$(PROCS) lean - rm -rf $(@D)/out/release/tests + make -C $(@D)/out/release -j$(PROCS) bin_lean mkdir -p extern/mathlib cp -u $(@D)/leanpkg/leanpkg.toml extern/mathlib touch $@ From c0cc30d6b85a7624f9cb1feb918cd591db9a8a95 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 15 Sep 2025 17:12:16 +0200 Subject: [PATCH 29/75] Makefile: tar: exclude some files from extraction that are unneeded --- Makefile | 6 +++--- notes | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c59b70d5..acf6d2bb 100644 --- a/Makefile +++ b/Makefile @@ -107,9 +107,11 @@ extern/%/.built: extern/%/.unpacked make -C $(@D) $($*_ENV) -j$(PROCS) touch $@ +dh_TAR_FLAG=--wildcards --exclude=*/docs --exclude=*/benchmarks --exclude=*/src/{exterminator,local,replicated} +sm_TAR_FLAG=--wildcards --exclude=*/{doc,paper,short-talk,talk,other-mallocs,benchmarks,tests} extern/%/.unpacked: archives/%.tar.gz mkdir -p $(@D) - tar -x --strip-components=1 --overwrite -f $< -C $(@D) + tar -x --strip-components=1 --overwrite -f $< -C $(@D) $($*_TAR_FLAG) touch $@ .PRECIOUS: archives/%.tar.gz @@ -122,7 +124,6 @@ archives/%.tar.gz: ######################################################################## #dh: uses cmake extern/dh/.configured: extern/dh/.unpacked - cd $(@D) && rm -rf ./benchmarks/ ./src/exterminator/ ./src/local/ ./src/replicated/ ./docs cmake -S $(@D)/src -B $(@D)/build touch $@ @@ -200,7 +201,6 @@ extern/scudo/.built: extern/scudo/.unpacked #sm: make, but a fix before extern/sm/.built: extern/sm/.unpacked - rm -rf ./$(@D)/doc ./$(@D)/paper ./$(@D)/short-talk ./$(@D)/talk sed -i "s/-Werror//" $(@D)/Makefile.include make -C $(@D)/release -j$(PROCS) ../release/lib/libsupermalloc.so diff --git a/notes b/notes index ca953831..6660b692 100644 --- a/notes +++ b/notes @@ -3,6 +3,7 @@ TODO versionize archives for that adapt CI-scripts to use Makefile we can give tar the list of files we want to extract or not extract + sm: check file exclusion list on x86, also choose minmal make target package installation document in README move to CI instead of build_bench_env.sh From 63877ec52687ea33363b5ec507765630834555a3 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 16 Sep 2025 07:42:58 +0200 Subject: [PATCH 30/75] sm: do not extract unneeded things --- Makefile | 3 ++- notes | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index acf6d2bb..fe745d9c 100644 --- a/Makefile +++ b/Makefile @@ -108,7 +108,7 @@ extern/%/.built: extern/%/.unpacked touch $@ dh_TAR_FLAG=--wildcards --exclude=*/docs --exclude=*/benchmarks --exclude=*/src/{exterminator,local,replicated} -sm_TAR_FLAG=--wildcards --exclude=*/{doc,paper,short-talk,talk,other-mallocs,benchmarks,tests} +sm_TAR_FLAG=--wildcards --exclude=*/{.junk,benchmarks,coverage,debug,doc,other-mallocs,paper,short-talk,talk,tests,tools} extern/%/.unpacked: archives/%.tar.gz mkdir -p $(@D) tar -x --strip-components=1 --overwrite -f $< -C $(@D) $($*_TAR_FLAG) @@ -203,6 +203,7 @@ extern/scudo/.built: extern/scudo/.unpacked extern/sm/.built: extern/sm/.unpacked sed -i "s/-Werror//" $(@D)/Makefile.include make -C $(@D)/release -j$(PROCS) ../release/lib/libsupermalloc.so + touch $@ #sn: cmake+ninja, builds in sn/release extern/sn/.built: extern/sn/build.ninja diff --git a/notes b/notes index 6660b692..a61e736c 100644 --- a/notes +++ b/notes @@ -2,8 +2,6 @@ TODO put archives in extra storage (and somehow map that for CI) versionize archives for that adapt CI-scripts to use Makefile - we can give tar the list of files we want to extract or not extract - sm: check file exclusion list on x86, also choose minmal make target package installation document in README move to CI instead of build_bench_env.sh From e70c117153f7e0396b42b0db74adf31810b6f06d Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 16 Sep 2025 14:03:52 +0200 Subject: [PATCH 31/75] Dependencies: document in README, start to adapt CI --- Dockerfile | 11 ++++++--- README.md | 71 +++++++++++++++++++++++++++++++++++++++++++++--------- notes | 2 +- 3 files changed, 68 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index fe54d991..27f2cff1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,12 @@ ARG platform=ubuntu FROM ubuntu:24.04 AS ubuntu - +RUN sudo apt-get install --no-install-recommends build-essential git gpg \ + g++ clang lld llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ + cmake python3 ruby ninja-build libtool autoconf sed ghostscript \ + time curl automake libatomic1 libgflags-dev libsnappy-dev \ + zlib1g-dev libbz2-dev liblz4-dev libzstd-dev libreadline-dev \ + pkg-config gawk util-linux bazel-bootstrap FROM fedora:latest AS fedora @@ -18,7 +23,7 @@ COPY . /mimalloc-bench WORKDIR /mimalloc-bench # Install dependencies -RUN ./build-bench-env.sh packages +# RUN ./build-bench-env.sh packages # Build benchmarks RUN ./build-bench-env.sh bench @@ -41,4 +46,4 @@ RUN ./build-bench-env.sh $allocator # Run benchmarks WORKDIR /mimalloc-bench/out/bench -RUN ../../bench.sh $allocator $benchs -r=$repeats \ No newline at end of file +RUN ../../bench.sh $allocator $benchs -r=$repeats diff --git a/README.md b/README.md index efce48f8..0bfb6201 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,8 @@ Collection of various benchmarks from the academic literature, together with automated scripts to pull specific versions of benchmark programs and allocators from Github and build them. -Due to the large variance in programs and allocators, the suite is currently -only developed for Unix-like systems, and specifically Ubuntu with `apt-get`, Fedora with `dnf`, -and macOS (for a limited set of allocators and benchmarks). The only system-installed allocator used is glibc's implementation that ships as part of Linux's libc. -All other allocators are downloaded and built as part of `build-bench-env.sh` -- -if you are looking to run these benchmarks on a different Linux distribution look at -the `setup_packages` function to see the packages required to build the full set of -allocators. - +All other allocators are downloaded and built with `make`. It is quite easy to add new benchmarks and allocator implementations -- please do so!. @@ -27,19 +20,73 @@ Enjoy, Daan - Note that all the code in the `bench` directory is not part of _mimalloc-bench_ as such, and all programs in the `bench` directory are governed under their own specific licenses and copyrights as detailed in their `README.md` (or `license.txt`) files. They are just included here for convenience. +# Dependencies +Some of the allocators and benchmarks need packages installed on the +system to be built and/or run. See the following collection for your +distribution: +
+ Ubuntu/Debian + +```bash +sudo apt-get install --no-install-recommends build-essential git gpg \ + g++ clang lld llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ + cmake python3 ruby ninja-build libtool autoconf sed ghostscript \ + time curl automake libatomic1 libgflags-dev libsnappy-dev \ + zlib1g-dev libbz2-dev liblz4-dev libzstd-dev libreadline-dev \ + pkg-config gawk util-linux bazel-bootstrap +``` +
+ +
+ Fedora + +```bash +sudo dnf --nodocs gcc-c++ clang lld llvm-devel unzip dos2unix bc \ + gmp-devel wget gawk cmake python3 ruby ninja-build libtool autoconf \ + git patch time sed ghostscript libatomic libstdc++ which gflags-devel \ + xz readline-devel snappy-devel +``` +
+ +
+ Alpine + +```bash +sudo apk add clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ + automake gawk samurai libtool git build-base linux-headers autoconf \ + util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev +``` +
+ +
+ ArchLinux + +```bash +sudo pacman -S dos2unix wget cmake ninja automake libtool time gmp sed \ + ghostscript bazelisk gflags snappy python-six +``` +
+ +
+ Mac OS + +```bash +brew install dos2unix wget cmake ninja automake libtool gnu-time gmp \ + mpir gnu-sed ghostscript bazelisk gflags snappy +``` +
# Benchmarking -The `build-bench-env.sh` script with the `all` argument will automatically pull -all needed benchmarks and allocators and build them in the `extern` directory: +`make all` will automatically pull all needed benchmarks and allocators +and build them in the `extern` directory: ``` -~/dev/mimalloc-bench> ./build-bench-env.sh all +~/dev/mimalloc-bench> make all ``` It starts installing packages and you will need to enter the sudo password. All other programs are build in the `mimalloc-bench/extern` directory. diff --git a/notes b/notes index a61e736c..547ec906 100644 --- a/notes +++ b/notes @@ -3,7 +3,7 @@ TODO versionize archives for that adapt CI-scripts to use Makefile package installation - document in README + document in README: bazel5 missing, rest is done move to CI instead of build_bench_env.sh Decisions made: From 69e446038393f664539ee593eccebd47087d18b0 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 16 Sep 2025 22:54:24 +0200 Subject: [PATCH 32/75] CI: make Ubuntu work --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 27f2cff1..1cffec06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,8 @@ ARG platform=ubuntu FROM ubuntu:24.04 AS ubuntu -RUN sudo apt-get install --no-install-recommends build-essential git gpg \ +RUN apt-get update +RUN apt-get install -y --no-install-recommends build-essential git gpg \ g++ clang lld llvm-dev unzip dos2unix linuxinfo bc libgmp-dev wget \ cmake python3 ruby ninja-build libtool autoconf sed ghostscript \ time curl automake libatomic1 libgflags-dev libsnappy-dev \ From 6003c37bf6a88716b03280400ecdce91c94efed3 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 25 Sep 2025 16:01:22 +0200 Subject: [PATCH 33/75] ported notes into github PR description --- notes | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 notes diff --git a/notes b/notes deleted file mode 100644 index 547ec906..00000000 --- a/notes +++ /dev/null @@ -1,28 +0,0 @@ -TODO - put archives in extra storage (and somehow map that for CI) - versionize archives for that - adapt CI-scripts to use Makefile - package installation - document in README: bazel5 missing, rest is done - move to CI instead of build_bench_env.sh - -Decisions made: - VERSIONS: demand versions instead of reporting them in the end - bench.sh needs to be adapted in the way it finds 'installed allocators' - switched from git cloning to grabbing a snapshot of the reqeuested version from github - dh: move build directory (dh/src/build -> dh/build) - lua: moved to extern benchmarks because it is built in extern/ - benchmarks: if something is updated, CMakeLists.txt needs to be touched (because bench/ is not part of the make logic at all) - that could be fixed by suppplying a Makefile in bench/, too. - sm: instead of default target, switched to building supermalloc only - linux: updated version to LTS series - lean: switched to smaller build target - -open Questions: - partial_checkout - lp Also, github does not serve archives on that repo. Re-add when solved. - scudo - pa: re-add, but smaller? https://chromium.googlesource.com/chromium/src/base/allocator/partition_allocator - build system uses depot_tools which come with problems, also does not compile even when installed - sed patches in Makefile: port to proper patch? - at least gd, hd, rp are patched with sed right now, we could port that to a patch From bc0936a0d55db0d0e81fe75e0e58479d28e2fb6d Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 25 Sep 2025 16:17:11 +0200 Subject: [PATCH 34/75] CI: switch build_bench_env to Makefile --- Dockerfile | 28 +++++++++++++++------------- Makefile | 2 -- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1cffec06..dd456257 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,11 +10,17 @@ RUN apt-get install -y --no-install-recommends build-essential git gpg \ pkg-config gawk util-linux bazel-bootstrap FROM fedora:latest AS fedora - +RUN dnf -y --quiet --nodocs install gcc-c++ clang lld llvm-devel unzip \ + dos2unix bc gmp-devel wget gawk cmake python3 ruby ninja-build libtool \ + autoconf git patch time sed ghostscript libatomic libstdc++ which \ + gflags-devel xz readline-devel snappy-devel FROM alpine:latest AS alpine +RUN apk update RUN apk add --no-cache bash - +RUN apk add -q clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ + automake gawk samurai libtool git build-base linux-headers autoconf \ + util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev FROM ${platform} AS bench-env @@ -23,17 +29,13 @@ RUN mkdir -p /mimalloc-bench COPY . /mimalloc-bench WORKDIR /mimalloc-bench -# Install dependencies -# RUN ./build-bench-env.sh packages - -# Build benchmarks -RUN ./build-bench-env.sh bench - -RUN ./build-bench-env.sh redis - -RUN ./build-bench-env.sh rocksdb -RUN ./build-bench-env.sh lean +RUN make benchmarks +RUN make redis +RUN make rocksdb +RUN make lean +RUN make lua +RUN make linux FROM bench-env AS benchmark @@ -43,7 +45,7 @@ ARG allocator=mi ARG benchs=cfrac ARG repeats=1 -RUN ./build-bench-env.sh $allocator +RUN make $allocator # Run benchmarks WORKDIR /mimalloc-bench/out/bench diff --git a/Makefile b/Makefile index fe745d9c..381c599f 100644 --- a/Makefile +++ b/Makefile @@ -79,8 +79,6 @@ bench/shbench/%.zip: @cd $(@D) && wget -nc --no-verbose http://www.microquill.com/smartheap/$($*_FILENAME) @(echo "$($*_SHA256SUM) $@" | $(SHA256SUM) --check --status) || { echo $(err_msg); exit 1; } -dependencies: - ######################################################################## # Environment flags for the individual make processes, may just be the # # respective target name. # From ffbdf87966a9024a8426dd33adc6ee7a46642b03 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 25 Sep 2025 16:20:08 +0200 Subject: [PATCH 35/75] Makefile: use make function instead of echo --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 381c599f..f3a36cf2 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ CXXFLAGS+=-march=native SUDO=sudo ifeq ($(shell whoami), root) - echo "running as root, avoid doing this if possible." - SUDO= +$(warning running as root, avoid doing this if possible.) +SUDO= endif DARWIN=no @@ -13,11 +13,11 @@ EXTSO=so SHA256SUM="sha256sum" ifeq ($(shell uname), Darwin) - DARWIN=yes - PROCS=$(shell sysctl -n hw.physicalcpu) - EXTSO=dylib - SHA256SUM=shasum -a 256 - export HOMEBREW_NO_EMOJI=1 +DARWIN=yes +PROCS=$(shell sysctl -n hw.physicalcpu) +EXTSO=dylib +SHA256SUM=shasum -a 256 +export HOMEBREW_NO_EMOJI=1 endif BENCHMARKS_EXTERN=lean linux lua redis rocksdb @@ -27,10 +27,10 @@ PDFDOC=extern/large.pdf # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) - ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) - ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) - # gd uses SSE on x86, but ARC4 on ARM - and arc4 needs a fix - gd_ENV := ARC4RNG=1 +ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) +ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) +# gd uses SSE on x86, but ARC4 on ARM - and arc4 needs a fix +gd_ENV := ARC4RNG=1 extern/gd/.built: extern/gd/.unpacked sed -i_orig 's/getentropy(/_getentropy(/g' $(@D)/rng/arc4random.{c,h} make -C $(@D) $(gd_ENV) -j$(PROCS) From 6cac134b67a4f3b1f55cd13fcb62b7abfff6c4ea Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 25 Sep 2025 16:51:18 +0200 Subject: [PATCH 36/75] Makefile: fix sha256sum for alpine --- Makefile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index f3a36cf2..ab378fbd 100644 --- a/Makefile +++ b/Makefile @@ -10,16 +10,22 @@ endif DARWIN=no PROCS=$(shell nproc) EXTSO=so -SHA256SUM="sha256sum" +SHA256SUM=sha256sum +SHA256SUM_FLAGS=-c --status ifeq ($(shell uname), Darwin) DARWIN=yes PROCS=$(shell sysctl -n hw.physicalcpu) EXTSO=dylib -SHA256SUM=shasum -a 256 +SHA256SUM=shasum +SHA256SUM_FLAGS=-a 256 export HOMEBREW_NO_EMOJI=1 endif +ifneq ($(shell grep -e 'ID=alpine' /etc/os-release),) +SHA256SUM_FLAGS=-c -s +endif + BENCHMARKS_EXTERN=lean linux lua redis rocksdb ALLOCS_TRIVIAL = ff fg iso je lf lt mesh mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 nomesh rp sc scudo sm sn tcg yal @@ -77,7 +83,7 @@ bench_FILENAME=shbench/bench.zip bench_SHA256SUM=506354d66b9eebef105d757e055bc55e8d4aea1e7b51faab3da35b0466c923a1 bench/shbench/%.zip: @cd $(@D) && wget -nc --no-verbose http://www.microquill.com/smartheap/$($*_FILENAME) - @(echo "$($*_SHA256SUM) $@" | $(SHA256SUM) --check --status) || { echo $(err_msg); exit 1; } + @(echo "$($*_SHA256SUM) $@" | $(SHA256SUM) $(SHA256SUM_FLAGS)) || { echo $(err_msg); exit 1; } ######################################################################## # Environment flags for the individual make processes, may just be the # From afc94bb88ca372eb898b7d7a4923a14284062d77 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 25 Sep 2025 16:52:57 +0200 Subject: [PATCH 37/75] Makefile: create extern/ if not already present --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index ab378fbd..6dd70105 100644 --- a/Makefile +++ b/Makefile @@ -55,6 +55,7 @@ benchmarks: bench/CMakeLists.txt bench/shbench/sh6bench-new.c bench/shbench/sh8b PDF_URL=https://raw.githubusercontent.com/geekaaron/Resources/master/resources/Writing_a_Simple_Operating_System--from_Scratch.pdf $(PDFDOC): + mkdir -p extern wget --no-verbose -O $(PDFDOC) $(PDF_URL) bench/shbench/sh6bench-new.c: bench/shbench/sh6bench.patch bench/shbench/sh6bench.c From 7f49ba081715543a46620179ec6f18a836be00b0 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 29 Sep 2025 13:19:17 +0200 Subject: [PATCH 38/75] CI: alpine: install bazel@testing --- Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile b/Dockerfile index dd456257..c9b608fb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,8 @@ RUN apk add --no-cache bash RUN apk add -q clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ automake gawk samurai libtool git build-base linux-headers autoconf \ util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +RUN apk add -q bazel@testing FROM ${platform} AS bench-env From cbfec9e350adb34e8d8579d6e8bb35822ba33910 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 29 Sep 2025 13:54:20 +0200 Subject: [PATCH 39/75] bench.sh: remove versioning info from benchmarks paths --- bench.sh | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/bench.sh b/bench.sh index 5be7d8cb..d4cb81ec 100755 --- a/bench.sh +++ b/bench.sh @@ -24,14 +24,6 @@ tests_run="" tests_exclude="" readonly tests_exclude_macos="sh6bench sh8bench redis" -# -------------------------------------------------------------------- -# benchmark versions -# -------------------------------------------------------------------- - -readonly version_redis=6.2.7 -readonly version_rocksdb=8.1.1 -readonly version_linux=6.5.1 - # -------------------------------------------------------------------- # Environment # -------------------------------------------------------------------- @@ -163,10 +155,10 @@ fi readonly luadir="$localdevdir/lua" readonly leandir="$localdevdir/lean" readonly leanmldir="$leandir/../mathlib" -readonly redis_dir="$localdevdir/redis-$version_redis/src" +readonly redis_dir="$localdevdir/redis/src" readonly pdfdoc="$localdevdir/large.pdf" -readonly rocksdb_dir="$localdevdir/rocksdb-$version_rocksdb" -readonly linux_dir="$localdevdir/linux-$version_linux" +readonly rocksdb_dir="$localdevdir/rocksdb" +readonly linux_dir="$localdevdir/linux" readonly spec_dir="$localdevdir/../../spec2017" readonly spec_base="base" From 956b8f8bff6cc2a1427f95a38d3324d2205cd356 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 29 Sep 2025 18:01:53 +0200 Subject: [PATCH 40/75] bench.sh: adapt mesh and nomesh library path to where it is built without cmake --- bench.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bench.sh b/bench.sh index d4cb81ec..19c0a054 100755 --- a/bench.sh +++ b/bench.sh @@ -102,9 +102,9 @@ alloc_lib_add "je" "$localdevdir/je/lib/libjemalloc$extso" alloc_lib_add "lf" "$localdevdir/lf/liblite-malloc-shared$extso" alloc_lib_add "lp" "$localdevdir/lp/Source/bmalloc/libpas/build-cmake-default/Release/libpas_lib$extso" alloc_lib_add "lt" "$localdevdir/lt/gnu.make.lib/libltalloc$extso" -alloc_lib_add "mesh" "$localdevdir/mesh/build/lib/libmesh$extso" +alloc_lib_add "mesh" "$localdevdir/mesh/bazel-bin/src/libmesh$extso" alloc_lib_add "mng" "$localdevdir/mng/libmallocng$extso" -alloc_lib_add "nomesh" "$localdevdir/nomesh/build/lib/libmesh$extso" +alloc_lib_add "nomesh" "$localdevdir/nomesh/bazel-bin/src/libmesh$extso" alloc_lib_add "pa" "$localdevdir/pa/partition_alloc_builder/out/Default/libpalib$extso" alloc_lib_add "rp" "$lib_rp" alloc_lib_add "sc" "$localdevdir/sc/out/Release/lib.target/libscalloc$extso" From 92cb87a06ea02fd2570d5e63df2947d7eb0124a4 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 14:23:31 +0200 Subject: [PATCH 41/75] CI: fix bazel on fedora as per https://github.com/bazelbuild/bazel/issues/26156 --- .github/workflows/all.yml | 5 +---- Dockerfile | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/all.yml b/.github/workflows/all.yml index 930eee13..099f3756 100644 --- a/.github/workflows/all.yml +++ b/.github/workflows/all.yml @@ -17,11 +17,8 @@ jobs: strategy: matrix: platform: [ubuntu, alpine, fedora] - exclude: - # Fedora is not currently building lean. - - platform: fedora fail-fast: false uses: ./.github/workflows/platform.yml name: ${{ matrix.platform }} with: - platform: ${{ matrix.platform }} \ No newline at end of file + platform: ${{ matrix.platform }} diff --git a/Dockerfile b/Dockerfile index c9b608fb..106700be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,6 +14,8 @@ RUN dnf -y --quiet --nodocs install gcc-c++ clang lld llvm-devel unzip \ dos2unix bc gmp-devel wget gawk cmake python3 ruby ninja-build libtool \ autoconf git patch time sed ghostscript libatomic libstdc++ which \ gflags-devel xz readline-devel snappy-devel +RUN dnf -y --quiet copr enable ohadm/bazel +RUN dnf -y --quiet --nodocs install bazel5 FROM alpine:latest AS alpine RUN apk update From 668ae5b75d1b52717cbed61ea2b6899ab6c610cf Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 14:39:15 +0200 Subject: [PATCH 42/75] CI: try bazel8 on fedora, see if the allocs can be build with that --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 106700be..8f5559ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ RUN dnf -y --quiet --nodocs install gcc-c++ clang lld llvm-devel unzip \ autoconf git patch time sed ghostscript libatomic libstdc++ which \ gflags-devel xz readline-devel snappy-devel RUN dnf -y --quiet copr enable ohadm/bazel -RUN dnf -y --quiet --nodocs install bazel5 +RUN dnf -y --quiet --nodocs install bazel8 FROM alpine:latest AS alpine RUN apk update From eb2444a9a2e8ecd97600fee477351ce29b321f15 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 15:22:52 +0200 Subject: [PATCH 43/75] Makefile: patch rocksdb on fedore (depends on #240), move target declaration up --- Makefile | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 6dd70105..d73be44d 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,25 @@ PDFDOC=extern/large.pdf ifeq ($(shell uname -m), aarch64) ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) +endif + +all: allocs benchmarks_all +allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) +benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) + +# rocksdb: needs a fix on fedora +ifneq ($(shell grep -e 'ID=fedora' /etc/os-release),) +extern/rocksdb/.built: extern/rocksdb/.patched + make -C $(@D) $($*_ENV) -j$(PROCS) + touch $@ + +extern/rocksdb/.patched: extern/rocksdb/.unpacked + cd $(@D) && patch -p1 -N -r- < ../../patches/rocksdb_build.patch + touch $@ +endif + +# TODO: Mac seems to report 'arm64' here +ifeq ($(shell uname -m), aarch64) # gd uses SSE on x86, but ARC4 on ARM - and arc4 needs a fix gd_ENV := ARC4RNG=1 extern/gd/.built: extern/gd/.unpacked @@ -45,10 +64,6 @@ endif .PHONY: all allocs benchmarks benchmarks_all benchmarks_big -all: allocs benchmarks_all -allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) -benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) - benchmarks: bench/CMakeLists.txt bench/shbench/sh6bench-new.c bench/shbench/sh8bench-new.c $(PDFDOC) cmake -B out/bench -S bench cmake --build out/bench -j $(PROCS) From eeb4546f063e71079d26b41d21ef5c54a14314d1 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 15:23:30 +0200 Subject: [PATCH 44/75] Makefile/sn: fix a dependency causing constant rebuilds --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index d73be44d..c10e6cb1 100644 --- a/Makefile +++ b/Makefile @@ -226,11 +226,11 @@ extern/sm/.built: extern/sm/.unpacked touch $@ #sn: cmake+ninja, builds in sn/release -extern/sn/.built: extern/sn/build.ninja +extern/sn/.built: extern/sn/release/build.ninja cd $(@D)/release && ninja libsnmallocshim.$(EXTSO) libsnmallocshim-checks.$(EXTSO) touch $@ -extern/sn/build.ninja: extern/sn/.unpacked +extern/sn/release/build.ninja: extern/sn/.unpacked env CXX=clang++ cmake -S $(@D) -B $(@D)/release -G Ninja -DCMAKE_BUILD_TYPE=Release #tbb: cmake to configure From 33958c7f5827d34a5b3ec950b660e4f47af6a40b Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 15:23:42 +0200 Subject: [PATCH 45/75] Makefile: fix a typo --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c10e6cb1..bce6301e 100644 --- a/Makefile +++ b/Makefile @@ -279,6 +279,6 @@ extern/lean/.built: extern/lean/.unpacked extern/lua/.built: extern/lua/.unpacked touch $@ -# llinux only needs to be fetched, not more. +# linux only needs to be fetched, not more. extern/linux/.built: extern/linux/.unpacked touch $@ From 8bd60f5bb42dbe17fd7e64158b2e053eee7ec963 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 15:42:27 +0200 Subject: [PATCH 46/75] Makefile: declare flags before using them --- Makefile | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index bce6301e..c7941f48 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,20 @@ ALLOCS_TRIVIAL = ff fg iso je lf lt mesh mng sg tbb tc ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 nomesh rp sc scudo sm sn tcg yal PDFDOC=extern/large.pdf +######################################################################## +# Environment flags for the individual make processes, may just be the # +# respective target name. # +######################################################################## + +fg_env=SSE2RNG=1 +iso_ENV=library +mesh_ENV=build +lf_ENV=liblite-malloc-shared.so +lt_ENV=-C gnu.make.lib +hd_ENV=-C src +redis_ENV=USE_JEMALLOC=no MALLOC=libc BUILD_TLS=no -C src +rocksdb_ENV=DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 db_bench + # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) @@ -101,19 +115,6 @@ bench/shbench/%.zip: @cd $(@D) && wget -nc --no-verbose http://www.microquill.com/smartheap/$($*_FILENAME) @(echo "$($*_SHA256SUM) $@" | $(SHA256SUM) $(SHA256SUM_FLAGS)) || { echo $(err_msg); exit 1; } -######################################################################## -# Environment flags for the individual make processes, may just be the # -# respective target name. # -######################################################################## - -fg_env=SSE2RNG=1 -iso_ENV=library -mesh_ENV=build -lf_ENV=liblite-malloc-shared.so -lt_ENV=-C gnu.make.lib -hd_ENV=-C src -redis_ENV=USE_JEMALLOC=no MALLOC=libc BUILD_TLS=no -C src -rocksdb_ENV=DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALLOC=1 db_bench ######################################################################## # ALLOCS: generic targets for the standard scheme: download, unpack, # # configure (nop for the standard), compile (using make). # From 590c9f70545370583229d860f7fdc972bf938c1a Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 15:43:06 +0200 Subject: [PATCH 47/75] README: document how to install bazel for fedora and alpine --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 0bfb6201..a47432ad 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,9 @@ sudo apt-get install --no-install-recommends build-essential git gpg \ sudo dnf --nodocs gcc-c++ clang lld llvm-devel unzip dos2unix bc \ gmp-devel wget gawk cmake python3 ruby ninja-build libtool autoconf \ git patch time sed ghostscript libatomic libstdc++ which gflags-devel \ - xz readline-devel snappy-devel + xz readline-devel snappy-devel dnf-plugins-core +sudo dnf copr -y enable ohadm/bazel +sudo dnf --nodocs install bazel8 ``` @@ -60,6 +62,8 @@ sudo dnf --nodocs gcc-c++ clang lld llvm-devel unzip dos2unix bc \ sudo apk add clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ automake gawk samurai libtool git build-base linux-headers autoconf \ util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev +sudo echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +sudo apk add -q bazel@testing ``` From 5638901109ee810728cf1c52ee5c12fd3255cd76 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Tue, 30 Sep 2025 20:03:13 +0200 Subject: [PATCH 48/75] Makefile: apply rocksdb-patch on every platform --- Makefile | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index c7941f48..d7bb23bd 100644 --- a/Makefile +++ b/Makefile @@ -55,17 +55,6 @@ all: allocs benchmarks_all allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) -# rocksdb: needs a fix on fedora -ifneq ($(shell grep -e 'ID=fedora' /etc/os-release),) -extern/rocksdb/.built: extern/rocksdb/.patched - make -C $(@D) $($*_ENV) -j$(PROCS) - touch $@ - -extern/rocksdb/.patched: extern/rocksdb/.unpacked - cd $(@D) && patch -p1 -N -r- < ../../patches/rocksdb_build.patch - touch $@ -endif - # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) # gd uses SSE on x86, but ARC4 on ARM - and arc4 needs a fix @@ -283,3 +272,12 @@ extern/lua/.built: extern/lua/.unpacked # linux only needs to be fetched, not more. extern/linux/.built: extern/linux/.unpacked touch $@ + +# rocksdb: needs patching +extern/rocksdb/.built: extern/rocksdb/.patched + make -C $(@D) $(rocksdb_ENV) -j$(PROCS) + touch $@ + +extern/rocksdb/.patched: extern/rocksdb/.unpacked + cd $(@D) && patch -p1 -N -r- < ../../patches/rocksdb_build.patch + touch $@ From 272eb707082c8666ee469d7234eda5c36560f838 Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Thu, 9 Oct 2025 08:45:30 +0200 Subject: [PATCH 49/75] MAkefile: fix a typo Co-authored-by: Matthew Parkinson --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d7bb23bd..b49ab412 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ PDFDOC=extern/large.pdf # respective target name. # ######################################################################## -fg_env=SSE2RNG=1 +fg_ENV=SSE2RNG=1 iso_ENV=library mesh_ENV=build lf_ENV=liblite-malloc-shared.so From da5a2c6bb75c3a8274fb4a7fd452ea040b2ae32e Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 1 Oct 2025 14:44:24 +0200 Subject: [PATCH 50/75] Makefile: rocksdb: use CLI switch of patch instead of manually cd'ing --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b49ab412..3d5eba88 100644 --- a/Makefile +++ b/Makefile @@ -279,5 +279,5 @@ extern/rocksdb/.built: extern/rocksdb/.patched touch $@ extern/rocksdb/.patched: extern/rocksdb/.unpacked - cd $(@D) && patch -p1 -N -r- < ../../patches/rocksdb_build.patch + patch -d $(@D) -p1 -N -r- < patches/rocksdb_build.patch touch $@ From 601eb22b488698e3f5f7d5b65194b475be5db6e6 Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Thu, 9 Oct 2025 14:00:20 +0200 Subject: [PATCH 51/75] Makefile: fix incorrect variable reference found by CoPilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b49ab412..1219c33c 100644 --- a/Makefile +++ b/Makefile @@ -206,7 +206,7 @@ extern/sc/build/gyp/gyp: extern/sc/.unpacked #scudo: native clang, in a sub-directory extern/scudo/.built: extern/scudo/.unpacked - cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$extso *.cpp -pthread + cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$(EXTSO) *.cpp -pthread touch $@ #sm: make, but a fix before From 72233eb218087ed333f3d9428af21d69f90199f3 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:06:39 +0200 Subject: [PATCH 52/75] Makefile: download archives less cryptic --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3d5eba88..01b91031 100644 --- a/Makefile +++ b/Makefile @@ -124,10 +124,18 @@ extern/%/.unpacked: archives/%.tar.gz tar -x --strip-components=1 --overwrite -f $< -C $(@D) $($*_TAR_FLAG) touch $@ +define parse_version = + $(eval $1_URL := $(shell grep "$(1):" VERSIONS | cut -d, -f3)) + $(eval $1_VERSION := $(shell grep "$(1):" VERSIONS | cut -d, -f2| tr -d ' ')) +endef + +$(foreach alloc,$(ALLOCS_NONTRIVIAL) $(ALLOCS_TRIVIAL),$(eval $(call parse_version,$(alloc)))) +$(foreach bench,$(BENCHMARKS_EXTERN),$(eval $(call parse_version,$(bench)))) + .PRECIOUS: archives/%.tar.gz archives/%.tar.gz: mkdir -p $(@D) - wget -O $@ $(shell grep "$*:" VERSIONS | cut -d, -f3)/archive/$(shell grep "$*:" VERSIONS | cut -d, -f2| tr -d ' ').tar.gz + wget -O $@ $($*_URL)/archive/$($*_VERSION).tar.gz ######################################################################## # ALLOCS: nontrivial cases From f3ba1fa884fb968a8010ecd18fdde2515ed4c1c1 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Wed, 1 Oct 2025 14:44:24 +0200 Subject: [PATCH 53/75] Makefile: rocksdb: use CLI switch of patch instead of manually cd'ing --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 1219c33c..e104f7fd 100644 --- a/Makefile +++ b/Makefile @@ -279,5 +279,5 @@ extern/rocksdb/.built: extern/rocksdb/.patched touch $@ extern/rocksdb/.patched: extern/rocksdb/.unpacked - cd $(@D) && patch -p1 -N -r- < ../../patches/rocksdb_build.patch + patch -d $(@D) -p1 -N -r- < patches/rocksdb_build.patch touch $@ From bc755b841d54e702adebfee1b7e0371d5e2f5674 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:06:39 +0200 Subject: [PATCH 54/75] Makefile: download archives less cryptic --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e104f7fd..5ac02fd8 100644 --- a/Makefile +++ b/Makefile @@ -124,10 +124,18 @@ extern/%/.unpacked: archives/%.tar.gz tar -x --strip-components=1 --overwrite -f $< -C $(@D) $($*_TAR_FLAG) touch $@ +define parse_version = + $(eval $1_URL := $(shell grep "$(1):" VERSIONS | cut -d, -f3)) + $(eval $1_VERSION := $(shell grep "$(1):" VERSIONS | cut -d, -f2| tr -d ' ')) +endef + +$(foreach alloc,$(ALLOCS_NONTRIVIAL) $(ALLOCS_TRIVIAL),$(eval $(call parse_version,$(alloc)))) +$(foreach bench,$(BENCHMARKS_EXTERN),$(eval $(call parse_version,$(bench)))) + .PRECIOUS: archives/%.tar.gz archives/%.tar.gz: mkdir -p $(@D) - wget -O $@ $(shell grep "$*:" VERSIONS | cut -d, -f3)/archive/$(shell grep "$*:" VERSIONS | cut -d, -f2| tr -d ' ').tar.gz + wget -O $@ $($*_URL)/archive/$($*_VERSION).tar.gz ######################################################################## # ALLOCS: nontrivial cases From 37b4f19fcaf4974a85f3c63c226556fe382a2ba9 Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Thu, 9 Oct 2025 22:11:21 +0200 Subject: [PATCH 55/75] README.md: fix sudo echo into file as suggested by CoPilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a47432ad..c5c34b58 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sudo dnf --nodocs install bazel8 sudo apk add clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ automake gawk samurai libtool git build-base linux-headers autoconf \ util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev -sudo echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" | sudo tee -a /etc/apk/repositories sudo apk add -q bazel@testing ``` From a7538c7465191e9a844eebdae3c3135fe1cd5626 Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Thu, 9 Oct 2025 14:00:20 +0200 Subject: [PATCH 56/75] Makefile: fix incorrect variable reference found by CoPilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 01b91031..5ac02fd8 100644 --- a/Makefile +++ b/Makefile @@ -214,7 +214,7 @@ extern/sc/build/gyp/gyp: extern/sc/.unpacked #scudo: native clang, in a sub-directory extern/scudo/.built: extern/scudo/.unpacked - cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$extso *.cpp -pthread + cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$(EXTSO) *.cpp -pthread touch $@ #sm: make, but a fix before From 71112be244be7ee718e193bd4113b9d8c5d054da Mon Sep 17 00:00:00 2001 From: derSteFfi Date: Thu, 9 Oct 2025 22:11:21 +0200 Subject: [PATCH 57/75] README.md: fix sudo echo into file as suggested by CoPilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a47432ad..c5c34b58 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ sudo dnf --nodocs install bazel8 sudo apk add clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ automake gawk samurai libtool git build-base linux-headers autoconf \ util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev -sudo echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories +echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" | sudo tee -a /etc/apk/repositories sudo apk add -q bazel@testing ``` From bfc5227cfdf15c96324b4f5068ba46a15b11f02e Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:34:10 +0200 Subject: [PATCH 58/75] MAkefile: fix dh source dir --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5ac02fd8..c1e89136 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ archives/%.tar.gz: ######################################################################## #dh: uses cmake extern/dh/.configured: extern/dh/.unpacked - cmake -S $(@D)/src -B $(@D)/build + cmake -S $(@D) -B $(@D)/build touch $@ extern/dh/.built: extern/dh/.configured From b173730868296cddf720c8e02ad6d46a84658b92 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:37:40 +0200 Subject: [PATCH 59/75] Makefile: fix sn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c1e89136..41993899 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ extern/sn/.built: extern/sn/release/build.ninja touch $@ extern/sn/release/build.ninja: extern/sn/.unpacked - env CXX=clang++ cmake -S $(@D) -B $(@D)/release -G Ninja -DCMAKE_BUILD_TYPE=Release + env CXX=clang++ cmake -S $(@D)/.. -B $(@D) -G Ninja -DCMAKE_BUILD_TYPE=Release #tbb: cmake to configure extern/tbb/.built: extern/tbb/.configured From 432467e64fd0655958bfd4cf3865ca703a453056 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:41:09 +0200 Subject: [PATCH 60/75] VERSIONS: re-switch to master for every allocator that was on master before --- VERSIONS | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/VERSIONS b/VERSIONS index dcb8bce5..dfa04088 100644 --- a/VERSIONS +++ b/VERSIONS @@ -1,31 +1,31 @@ -dh: master, cfe016b, https://github.com/emeryberger/DieHard -ff: master, 2f24ecf, https://github.com/bwickman97/ffmalloc -fg: master, bfdf6d9, https://github.com/UTSASRG/FreeGuard -gd: master, 9e85978, https://github.com/UTSASRG/Guarder +dh: master, master, https://github.com/emeryberger/DieHard +ff: master, master, https://github.com/bwickman97/ffmalloc +fg: master, master, https://github.com/UTSASRG/FreeGuard +gd: master, master, https://github.com/UTSASRG/Guarder hd: 6577c22b, 6577c22, https://github.com/emeryberger/Hoard hm: 11, 995ce07, https://github.com/GrapheneOS/hardened_malloc iso: 1.2.5, 2670d5f, https://github.com/struct/isoalloc je: 5.3.0, 54eaed1d, https://github.com/jemalloc/jemalloc lean: 21d264a66d53b0a910178ae7d9529cb5886a39b6, 21d264a66, https://github.com/leanprover-community/lean -lf: master, 915f51b, https://github.com/Begun/lockfree-malloc +lf: master, master, https://github.com/Begun/lockfree-malloc linux: 6.6.105, fe9731e1, https://github.com/torvalds/linux -lp: main, 08f06e1d, https://github.com/WebKit/WebKit -lt: master, f2d7df0, https://github.com/r-lyeh-archived/ltalloc +lp: main, main, https://github.com/WebKit/WebKit +lt: master, master, https://github.com/r-lyeh-archived/ltalloc lua: v5.4.7, 1ab3208a, https://github.com/lua/lua -mesh: master, d45d6de, https://github.com/plasma-umass/mesh +mesh: master, master, https://github.com/plasma-umass/mesh mi2: v2.1.2, 43ce4bd7, https://github.com/microsoft/mimalloc mi: v1.8.2, b66e3214, https://github.com/microsoft/mimalloc -mng: master, 2ed5881, https://github.com/richfelker/mallocng-draft -pa: main, e6c10be, https://github.com/1c3t3a/partition_alloc_builder +mng: master, master, https://github.com/richfelker/mallocng-draft +pa: main, main, https://github.com/1c3t3a/partition_alloc_builder redis: 6.2.7, e6f6709, https://github.com/redis/redis rocksdb: 8.1.1, 6a43615, https://github.com/facebook/rocksdb rp: 1.4.5, e4393ff, https://github.com/mjansson/rpmalloc -sc: master, 9ef491c, https://github.com/cksystemsgroup/scalloc +sc: master, master, https://github.com/cksystemsgroup/scalloc scudo: main, 30d3bb59, https://github.com/llvm/llvm-project -sg: master, 6d7d065, https://github.com/ssrg-vt/SlimGuard -sm: master, 709663f, https://github.com/kuszmaul/SuperMalloc +sg: master, master, https://github.com/ssrg-vt/SlimGuard +sm: master, master, https://github.com/kuszmaul/SuperMalloc sn: 0.7.1, 32495fd, https://github.com/Microsoft/snmalloc tbb: v2021.9.0, a00cc3b8, https://github.com/oneapi-src/oneTBB tcg: 98fd24303c7b5ef5e30da625f11fb623a5e038b6, 98fd2430, https://github.com/google/tcmalloc tc: gperftools-2.16.90, 83edb60, https://github.com/gperftools/gperftools -yal: main, d077646, https://github.com/jorisgeer/yalloc +yal: main, main, https://github.com/jorisgeer/yalloc From 79118be3da4c784834b21c90c64b2b6875a9546d Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:34:10 +0200 Subject: [PATCH 61/75] MAkefile: fix dh source dir --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 5ac02fd8..c1e89136 100644 --- a/Makefile +++ b/Makefile @@ -142,7 +142,7 @@ archives/%.tar.gz: ######################################################################## #dh: uses cmake extern/dh/.configured: extern/dh/.unpacked - cmake -S $(@D)/src -B $(@D)/build + cmake -S $(@D) -B $(@D)/build touch $@ extern/dh/.built: extern/dh/.configured From 1c16750a7102f1ca6dcb3efe763981853131a1bc Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:37:40 +0200 Subject: [PATCH 62/75] Makefile: fix sn --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c1e89136..41993899 100644 --- a/Makefile +++ b/Makefile @@ -229,7 +229,7 @@ extern/sn/.built: extern/sn/release/build.ninja touch $@ extern/sn/release/build.ninja: extern/sn/.unpacked - env CXX=clang++ cmake -S $(@D) -B $(@D)/release -G Ninja -DCMAKE_BUILD_TYPE=Release + env CXX=clang++ cmake -S $(@D)/.. -B $(@D) -G Ninja -DCMAKE_BUILD_TYPE=Release #tbb: cmake to configure extern/tbb/.built: extern/tbb/.configured From df23d8d0c615a2f51cf9a9e0bf254c804c1d591d Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:41:09 +0200 Subject: [PATCH 63/75] VERSIONS: re-switch to master for every allocator that was on master before --- VERSIONS | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/VERSIONS b/VERSIONS index dcb8bce5..dfa04088 100644 --- a/VERSIONS +++ b/VERSIONS @@ -1,31 +1,31 @@ -dh: master, cfe016b, https://github.com/emeryberger/DieHard -ff: master, 2f24ecf, https://github.com/bwickman97/ffmalloc -fg: master, bfdf6d9, https://github.com/UTSASRG/FreeGuard -gd: master, 9e85978, https://github.com/UTSASRG/Guarder +dh: master, master, https://github.com/emeryberger/DieHard +ff: master, master, https://github.com/bwickman97/ffmalloc +fg: master, master, https://github.com/UTSASRG/FreeGuard +gd: master, master, https://github.com/UTSASRG/Guarder hd: 6577c22b, 6577c22, https://github.com/emeryberger/Hoard hm: 11, 995ce07, https://github.com/GrapheneOS/hardened_malloc iso: 1.2.5, 2670d5f, https://github.com/struct/isoalloc je: 5.3.0, 54eaed1d, https://github.com/jemalloc/jemalloc lean: 21d264a66d53b0a910178ae7d9529cb5886a39b6, 21d264a66, https://github.com/leanprover-community/lean -lf: master, 915f51b, https://github.com/Begun/lockfree-malloc +lf: master, master, https://github.com/Begun/lockfree-malloc linux: 6.6.105, fe9731e1, https://github.com/torvalds/linux -lp: main, 08f06e1d, https://github.com/WebKit/WebKit -lt: master, f2d7df0, https://github.com/r-lyeh-archived/ltalloc +lp: main, main, https://github.com/WebKit/WebKit +lt: master, master, https://github.com/r-lyeh-archived/ltalloc lua: v5.4.7, 1ab3208a, https://github.com/lua/lua -mesh: master, d45d6de, https://github.com/plasma-umass/mesh +mesh: master, master, https://github.com/plasma-umass/mesh mi2: v2.1.2, 43ce4bd7, https://github.com/microsoft/mimalloc mi: v1.8.2, b66e3214, https://github.com/microsoft/mimalloc -mng: master, 2ed5881, https://github.com/richfelker/mallocng-draft -pa: main, e6c10be, https://github.com/1c3t3a/partition_alloc_builder +mng: master, master, https://github.com/richfelker/mallocng-draft +pa: main, main, https://github.com/1c3t3a/partition_alloc_builder redis: 6.2.7, e6f6709, https://github.com/redis/redis rocksdb: 8.1.1, 6a43615, https://github.com/facebook/rocksdb rp: 1.4.5, e4393ff, https://github.com/mjansson/rpmalloc -sc: master, 9ef491c, https://github.com/cksystemsgroup/scalloc +sc: master, master, https://github.com/cksystemsgroup/scalloc scudo: main, 30d3bb59, https://github.com/llvm/llvm-project -sg: master, 6d7d065, https://github.com/ssrg-vt/SlimGuard -sm: master, 709663f, https://github.com/kuszmaul/SuperMalloc +sg: master, master, https://github.com/ssrg-vt/SlimGuard +sm: master, master, https://github.com/kuszmaul/SuperMalloc sn: 0.7.1, 32495fd, https://github.com/Microsoft/snmalloc tbb: v2021.9.0, a00cc3b8, https://github.com/oneapi-src/oneTBB tcg: 98fd24303c7b5ef5e30da625f11fb623a5e038b6, 98fd2430, https://github.com/google/tcmalloc tc: gperftools-2.16.90, 83edb60, https://github.com/gperftools/gperftools -yal: main, d077646, https://github.com/jorisgeer/yalloc +yal: main, main, https://github.com/jorisgeer/yalloc From dcbb102857aaee5689e708baa0ceff108125c41a Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 22:57:01 +0200 Subject: [PATCH 64/75] Makefile re-unify the allocs --- Makefile | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 41993899..a50bfb3e 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,7 @@ SHA256SUM_FLAGS=-c -s endif BENCHMARKS_EXTERN=lean linux lua redis rocksdb -ALLOCS_TRIVIAL = ff fg iso je lf lt mesh mng sg tbb tc -ALLOCS_NONTRIVIAL = dh gd hd hm mi mi2 nomesh rp sc scudo sm sn tcg yal +ALLOCS = dh ff fg gd hd hm iso je lf lt mesh mi mi2 mng nomesh rp sc scudo sg sm sn tbb tc tcg yal PDFDOC=extern/large.pdf ######################################################################## @@ -47,12 +46,11 @@ rocksdb_ENV=DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALL # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) -ALLOCS_TRIVIAL := $(filter-out fg mesh lt, $(ALLOCS_TRIVIAL)) -ALLOCS_NONTRIVIAL := $(filter-out nomesh sc sm, $(ALLOCS_NONTRIVIAL)) +ALLOCS := $(filter-out fg lt mesh nomesh sc sm, $(ALLOCS_TRIVIAL)) endif all: allocs benchmarks_all -allocs: $(ALLOCS_TRIVIAL) $(ALLOCS_NONTRIVIAL) +allocs: $(ALLOCS) benchmarks_all: benchmarks $(BENCHMARKS_EXTERN) # TODO: Mac seems to report 'arm64' here @@ -110,8 +108,8 @@ bench/shbench/%.zip: ######################################################################## # Todo: check for the big benchmarks -$(ALLOCS_TRIVIAL) $(BENCHMARKS_EXTERN): %: extern/%/.built -$(ALLOCS_NONTRIVIAL): %: extern/%/.built +$(BENCHMARKS_EXTERN): %: extern/%/.built +$(ALLOCS): %: extern/%/.built extern/%/.built: extern/%/.unpacked make -C $(@D) $($*_ENV) -j$(PROCS) @@ -129,7 +127,7 @@ define parse_version = $(eval $1_VERSION := $(shell grep "$(1):" VERSIONS | cut -d, -f2| tr -d ' ')) endef -$(foreach alloc,$(ALLOCS_NONTRIVIAL) $(ALLOCS_TRIVIAL),$(eval $(call parse_version,$(alloc)))) +$(foreach alloc,$(ALLOCS),$(eval $(call parse_version,$(alloc)))) $(foreach bench,$(BENCHMARKS_EXTERN),$(eval $(call parse_version,$(bench)))) .PRECIOUS: archives/%.tar.gz @@ -149,7 +147,7 @@ extern/dh/.built: extern/dh/.configured cmake --build $(@D)/build -j $(PROCS) touch $@ -#hd: fix in Makefile. If later ported into a patch, hd can be reintegrated with ALLOCS_TRIVIAL +#hd: fix in Makefile. extern/hd/.built: extern/hd/.unpacked sed -i_orig 's/-arch arm64e//g' $(@D)/src/GNUmakefile sed -i_orig 's/-arch arm64//g' $(@D)/src/GNUmakefile From d987332ed464e5e2db526bc79bb7dbd6bc1ec144 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Thu, 9 Oct 2025 13:45:09 +0200 Subject: [PATCH 65/75] Makefile: shasum: correct flags on MacOS --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a50bfb3e..7f8d7365 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ ifeq ($(shell uname), Darwin) DARWIN=yes PROCS=$(shell sysctl -n hw.physicalcpu) EXTSO=dylib -SHA256SUM=shasum -SHA256SUM_FLAGS=-a 256 +SHA256SUM=shasum -a 256 +SHA256SUM_FLAGS=-c -s export HOMEBREW_NO_EMOJI=1 endif From d9c965ebde5f6e3f7ca9bf13b4b6098ad3394e1e Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 09:55:25 +0200 Subject: [PATCH 66/75] Revert "MAkefile: fix dh source dir" This reverts commit 79118be3da4c784834b21c90c64b2b6875a9546d. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7f8d7365..52eddac5 100644 --- a/Makefile +++ b/Makefile @@ -140,7 +140,7 @@ archives/%.tar.gz: ######################################################################## #dh: uses cmake extern/dh/.configured: extern/dh/.unpacked - cmake -S $(@D) -B $(@D)/build + cmake -S $(@D)/src -B $(@D)/build touch $@ extern/dh/.built: extern/dh/.configured From 40da63ee4ae8cff053cbeb91ec5499e4857dd0dd Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 09:56:15 +0200 Subject: [PATCH 67/75] Makefile: fix a remaining rename of ALLOCS --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52eddac5..d72829f6 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ rocksdb_ENV=DISABLE_WARNING_AS_ERROR=1 DISABLE_JEMALLOC=1 ROCKSDB_DISABLE_TCMALL # TODO: Mac seems to report 'arm64' here ifeq ($(shell uname -m), aarch64) -ALLOCS := $(filter-out fg lt mesh nomesh sc sm, $(ALLOCS_TRIVIAL)) +ALLOCS := $(filter-out fg lt mesh nomesh sc sm, $(ALLOCS)) endif all: allocs benchmarks_all From 209c65bb109a0e7d1d0e526d42a54e4d9446bf17 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 13:20:24 +0200 Subject: [PATCH 68/75] Makefile: re-add lp --- Makefile | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d72829f6..27afc8da 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ PROCS=$(shell nproc) EXTSO=so SHA256SUM=sha256sum SHA256SUM_FLAGS=-c --status +ORIG= ifeq ($(shell uname), Darwin) DARWIN=yes @@ -20,6 +21,7 @@ EXTSO=dylib SHA256SUM=shasum -a 256 SHA256SUM_FLAGS=-c -s export HOMEBREW_NO_EMOJI=1 +ORIG=_orig endif ifneq ($(shell grep -e 'ID=alpine' /etc/os-release),) @@ -27,7 +29,7 @@ SHA256SUM_FLAGS=-c -s endif BENCHMARKS_EXTERN=lean linux lua redis rocksdb -ALLOCS = dh ff fg gd hd hm iso je lf lt mesh mi mi2 mng nomesh rp sc scudo sg sm sn tbb tc tcg yal +ALLOCS = dh ff fg gd hd hm iso je lf lp lt mesh mi mi2 mng nomesh rp sc scudo sg sm sn tbb tc tcg yal PDFDOC=extern/large.pdf ######################################################################## @@ -170,6 +172,18 @@ extern/je/.built: extern/je/config.status extern/je/config.status: extern/je/.unpacked cd $(@D) && ./autogen.sh --enable-doc=no --enable-static=no --disable-stats +# lp: partial checkout and some fixes +extern/lp/.built: extern/lp/.unpacked + cd $(@D)/Source/bmalloc/libpas && CC=clang CXX=clang++ LDFLAGS='-lpthread -latomic -pthread' bash ./build.sh -s cmake -v default -t pas_lib + touch $@ + +extern/lp/.unpacked: + git clone --depth 1 --single-branch -b main --sparse --filter=blob:none https://github.com/WebKit/WebKit $(@D) + cd $(@D) && git sparse-checkout add Source/bmalloc/libpas + cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/extra_cmake_options=""/extra_cmake_options="-D_GNU_SOURCE=1"/' build.sh + cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/cmake --build $build_dir --parallel/cmake --build $build_dir --target pas_lib --parallel/' build.sh + touch $@ + # mi,mi2: cmake, and 3 different variants extern/mi/.built extern/mi2/.built: extern/%/.built: extern/%/.unpacked cmake -S $(@D) -B $(@D)/out/release From 5db0fa34adcea98013dca039a6c18bf8097b6c56 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 14:57:17 +0200 Subject: [PATCH 69/75] Makefile: lp: correctly quote "$" in sed line --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 27afc8da..c58e5d79 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ extern/lp/.unpacked: git clone --depth 1 --single-branch -b main --sparse --filter=blob:none https://github.com/WebKit/WebKit $(@D) cd $(@D) && git sparse-checkout add Source/bmalloc/libpas cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/extra_cmake_options=""/extra_cmake_options="-D_GNU_SOURCE=1"/' build.sh - cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/cmake --build $build_dir --parallel/cmake --build $build_dir --target pas_lib --parallel/' build.sh + cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/cmake --build $$build_dir --parallel/cmake --build $$build_dir --target pas_lib --parallel/' build.sh touch $@ # mi,mi2: cmake, and 3 different variants From 72d295728b070455791175bdd5caf96539d497ed Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 15:01:23 +0200 Subject: [PATCH 70/75] Makefile: do not interfere with CFLAGS, let the "subprojects" do that --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index c58e5d79..863cc863 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,3 @@ -CFLAGS+=-march=native -CXXFLAGS+=-march=native - SUDO=sudo ifeq ($(shell whoami), root) $(warning running as root, avoid doing this if possible.) From 39f18188735443f69f360547f234b7d19ba2b2ea Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 15:15:40 +0200 Subject: [PATCH 71/75] Makefile: fix scudo --- Makefile | 9 +++++++-- VERSIONS | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 863cc863..d64725ed 100644 --- a/Makefile +++ b/Makefile @@ -221,9 +221,14 @@ extern/sc/Makefile: extern/sc/build/gyp/gyp extern/sc/build/gyp/gyp: extern/sc/.unpacked cd extern/sc && tools/make_deps.sh -#scudo: native clang, in a sub-directory +#scudo: partial checkout, native clang, in a sub-directory +extern/scudo/.unpacked: + git clone --depth 1 --single-branch -b $(scudo_VERSION) --sparse --filter=blob:none $(scudo_URL) $(@D) + cd $(@D) && git sparse-checkout add compiler-rt/lib/scudo/standalone + touch $@ + extern/scudo/.built: extern/scudo/.unpacked - cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -std=c++17 -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -msse4.2 -O3 -I include -shared -o libscudo$(EXTSO) *.cpp -pthread + cd $(@D)/compiler-rt/lib/scudo/standalone && clang++ -flto -fuse-ld=lld -fPIC -fno-exceptions $(CXXFLAGS) -fno-rtti -fvisibility=internal -O3 -I include -shared -o libscudo.$(EXTSO) *.cpp touch $@ #sm: make, but a fix before diff --git a/VERSIONS b/VERSIONS index dfa04088..a7af451b 100644 --- a/VERSIONS +++ b/VERSIONS @@ -21,7 +21,7 @@ redis: 6.2.7, e6f6709, https://github.com/redis/redis rocksdb: 8.1.1, 6a43615, https://github.com/facebook/rocksdb rp: 1.4.5, e4393ff, https://github.com/mjansson/rpmalloc sc: master, master, https://github.com/cksystemsgroup/scalloc -scudo: main, 30d3bb59, https://github.com/llvm/llvm-project +scudo: main, main, https://github.com/llvm/llvm-project sg: master, master, https://github.com/ssrg-vt/SlimGuard sm: master, master, https://github.com/kuszmaul/SuperMalloc sn: 0.7.1, 32495fd, https://github.com/Microsoft/snmalloc From 8543d4152b6a142f84c889c9b4e1b36a26e1b646 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Fri, 10 Oct 2025 15:16:14 +0200 Subject: [PATCH 72/75] Makefile: parse version info for scudo and lp the way it is done for everyone else --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d64725ed..2a538c8d 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,7 @@ extern/lp/.built: extern/lp/.unpacked touch $@ extern/lp/.unpacked: - git clone --depth 1 --single-branch -b main --sparse --filter=blob:none https://github.com/WebKit/WebKit $(@D) + git clone --depth 1 --single-branch -b $(lp_VERSION) --sparse --filter=blob:none $(lp_URL) $(@D) cd $(@D) && git sparse-checkout add Source/bmalloc/libpas cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/extra_cmake_options=""/extra_cmake_options="-D_GNU_SOURCE=1"/' build.sh cd $(@D)/Source/bmalloc/libpas && sed -i $(ORIG) 's/cmake --build $$build_dir --parallel/cmake --build $$build_dir --target pas_lib --parallel/' build.sh From 2d0082e8f07273e750eb8defc83e6d7548bd612c Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 13 Oct 2025 15:19:12 +0200 Subject: [PATCH 73/75] CI: ubuntu: install bazel the recommended way --- Dockerfile | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 8f5559ec..e2718a2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,7 +7,14 @@ RUN apt-get install -y --no-install-recommends build-essential git gpg \ cmake python3 ruby ninja-build libtool autoconf sed ghostscript \ time curl automake libatomic1 libgflags-dev libsnappy-dev \ zlib1g-dev libbz2-dev liblz4-dev libzstd-dev libreadline-dev \ - pkg-config gawk util-linux bazel-bootstrap + pkg-config gawk util-linux +# Install bazel +RUN apt-get install -y --no-install-recommends openjdk-8-jdk apt-transport-https +RUN add-apt-repository ppa:webupd8team/java +RUN apt-get update && apt-get install oracle-java8-installer +RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list +RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - +RUN apt-get update && apt-get install bazel FROM fedora:latest AS fedora RUN dnf -y --quiet --nodocs install gcc-c++ clang lld llvm-devel unzip \ From a92744a371c79e0cd06d3b49cd94c09e64cdd3a6 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 13 Oct 2025 15:26:42 +0200 Subject: [PATCH 74/75] CI: ubuntu: bazel: stick to the install method from build_bench_env.sh --- Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e2718a2e..70d688cf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,11 +9,10 @@ RUN apt-get install -y --no-install-recommends build-essential git gpg \ zlib1g-dev libbz2-dev liblz4-dev libzstd-dev libreadline-dev \ pkg-config gawk util-linux # Install bazel -RUN apt-get install -y --no-install-recommends openjdk-8-jdk apt-transport-https -RUN add-apt-repository ppa:webupd8team/java -RUN apt-get update && apt-get install oracle-java8-installer +RUN apt-get install -y --no-install-recommends apt-transport-https curl gnupg +RUN curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor > bazel.gpg +RUN mv bazel.gpg /etc/apt/trusted.gpg.d/bazel.gpg RUN echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" > /etc/apt/sources.list.d/bazel.list -RUN curl https://bazel.build/bazel-release.pub.gpg | apt-key add - RUN apt-get update && apt-get install bazel FROM fedora:latest AS fedora From 007b144c3ce935c4666a112e5f68f1498f8be5d8 Mon Sep 17 00:00:00 2001 From: Stephan Bauroth Date: Mon, 13 Oct 2025 15:53:43 +0200 Subject: [PATCH 75/75] CI: alpine: install bazel the way build_bench_env.sh did --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 70d688cf..3c86bc89 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,12 +24,12 @@ RUN dnf -y --quiet copr enable ohadm/bazel RUN dnf -y --quiet --nodocs install bazel8 FROM alpine:latest AS alpine +RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories RUN apk update RUN apk add --no-cache bash RUN apk add -q clang lld unzip dos2unix bc gmp-dev wget cmake python3 \ automake gawk samurai libtool git build-base linux-headers autoconf \ util-linux sed ghostscript libatomic gflags-dev readline-dev snappy-dev -RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories RUN apk add -q bazel@testing FROM ${platform} AS bench-env