Skip to content

Commit 77fc76d

Browse files
auxtenauxten
authored
Merge pull request #274 from chdb-io/buildLinuxArm64
Fix jemalloc background_thread caused assert failed
2 parents e326128 + 84cd9e2 commit 77fc76d

File tree

3 files changed

+185
-116
lines changed

3 files changed

+185
-116
lines changed

‎.github/workflows/build_linux_arm64_wheels-gh.yml

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,180 @@
1+
name: Build Linux arm64
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
TAG_NAME:
7+
description: 'Release Version Tag'
8+
required: true
9+
release:
10+
types: [created]
11+
push:
12+
branches:
13+
- main
14+
paths-ignore:
15+
- '**/*.md'
16+
pull_request:
17+
branches:
18+
- main
19+
paths-ignore:
20+
- '**/*.md'
21+
22+
23+
jobs:
24+
build_wheels_linux:
25+
name: ${{ matrix.os }} py${{ matrix.python-version }}
26+
runs-on: GH-Linux-ARM64
27+
strategy:
28+
fail-fast: false
29+
matrix:
30+
os: [ ubuntu-24.04 ]
31+
python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ]
32+
# python-version: [ "3.7" ]
33+
env:
34+
RUNNER_OS: ${{ matrix.os }}
35+
PYTHON_VERSION: ${{ matrix.python-version }}
36+
steps:
37+
- name: Install python
38+
uses: actions/setup-python@v4
39+
with:
40+
python-version: "${{ matrix.python-version }}"
41+
- name: Install clang++ for Ubuntu
42+
if: matrix.os == 'ubuntu-24.04'
43+
run: |
44+
pwd
45+
uname -a
46+
wget https://apt.llvm.org/llvm.sh
47+
chmod +x llvm.sh
48+
sudo ./llvm.sh 18
49+
which clang++-18
50+
clang++-18 --version
51+
sudo apt-get install -y make cmake ccache ninja-build yasm gawk wget
52+
ccache -s
53+
- name: Update git
54+
run: |
55+
sudo add-apt-repository ppa:git-core/ppa -y
56+
sudo apt-get update
57+
sudo apt-get install -y git
58+
git --version
59+
- uses: actions/checkout@v3
60+
with:
61+
fetch-depth: 0
62+
- name: Restore submodules cache
63+
uses: actions/cache/restore@v3
64+
id: cache
65+
with:
66+
path: |
67+
contrib/**
68+
!contrib/*-cmake
69+
!**/CMakeLists.txt
70+
key: |
71+
submodule-${{ hashFiles('.gitmodules') }}
72+
- name: Update submodules if cache miss
73+
if: steps.cache.outputs.cache-hit != 'true'
74+
run: |
75+
git submodule update --init --recursive --jobs 4
76+
- name: Save submodules cache
77+
if: steps.cache.outputs.cache-hit != 'true'
78+
uses: actions/cache/save@v3
79+
with:
80+
path: |
81+
contrib/**
82+
!contrib/*-cmake
83+
!**/CMakeLists.txt
84+
key: |
85+
submodule-${{ hashFiles('.gitmodules') }}
86+
- name: ccache
87+
uses: hendrikmuhs/ccache-action@v1.2
88+
with:
89+
key: ${{ matrix.os }}
90+
max-size: 5G
91+
append-timestamp: true
92+
- name: remove old clang and link clang-18 to clang
93+
if: matrix.os == 'ubuntu-24.04'
94+
run: |
95+
sudo rm -f /usr/bin/clang || true
96+
sudo ln -s /usr/bin/clang-18 /usr/bin/clang
97+
sudo rm -f /usr/bin/clang++ || true
98+
sudo ln -s /usr/bin/clang++-18 /usr/bin/clang++
99+
which clang++
100+
clang++ --version
101+
- name: Run chdb/build.sh
102+
run: |
103+
python3 -m pip install pybind11 setuptools
104+
export CC=/usr/bin/clang
105+
export CXX=/usr/bin/clang++
106+
bash ./chdb/build.sh
107+
python3 -m pip install pandas pyarrow
108+
bash -x ./chdb/test_smoke.sh
109+
continue-on-error: false
110+
- name: Run libchdb stub in examples dir
111+
run: |
112+
bash -x ./examples/runStub.sh
113+
- name: Check ccache statistics
114+
run: |
115+
ccache -s
116+
ls -lh chdb
117+
df -h
118+
- name: Install dependencies for building wheels
119+
run: python3 -m pip install -U pip tox pybind11 twine setuptools wheel
120+
- name: Build wheels
121+
run: |
122+
export CC=/usr/bin/clang
123+
export CXX=/usr/bin/clang++
124+
make wheel
125+
- name: Install patchelf from github
126+
run: |
127+
wget https://github.com/NixOS/patchelf/releases/download/0.18.0/patchelf-0.18.0-aarch64.tar.gz -O patchelf.tar.gz
128+
tar -xvf patchelf.tar.gz
129+
sudo cp bin/patchelf /usr/bin/
130+
sudo chmod +x /usr/bin/patchelf
131+
patchelf --version
132+
- name: Audit wheels
133+
run: |
134+
python3 -m pip install auditwheel
135+
auditwheel -v repair -w dist/ --plat manylinux_2_17_aarch64 dist/*.whl
136+
continue-on-error: false
137+
- name: Show files
138+
run: |
139+
# e.g: remove chdb-0.11.4-cp310-cp310-linux_aarch64.whl, keep chdb-0.11.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl
140+
sudo rm -f dist/*linux_aarch64.whl
141+
ls -lh dist
142+
shell: bash
143+
- name: Run tests
144+
run: |
145+
python3 -m pip install dist/*.whl
146+
python3 -m pip install pandas pyarrow psutil
147+
python3 -c "import chdb; res = chdb.query('select 1112222222,555', 'CSV'); print(res)"
148+
make test
149+
continue-on-error: false
150+
- name: Upload wheels to release
151+
if: startsWith(github.ref, 'refs/tags/v')
152+
run: |
153+
gh release upload ${{ github.ref_name }} dist/*.whl --clobber
154+
env:
155+
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
156+
- name: Packege libchdb.so
157+
if: matrix.python-version == '3.12'
158+
run: |
159+
cp programs/local/chdb.h chdb.h
160+
tar -czvf linux-aarch64-libchdb.tar.gz libchdb.so chdb.h
161+
- name: Upload libchdb.so to release
162+
if: startsWith(github.ref, 'refs/tags/v') && matrix.python-version == '3.12'
163+
run: |
164+
gh release upload ${{ github.ref_name }} linux-aarch64-libchdb.tar.gz --clobber
165+
env:
166+
GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
167+
- uses: actions/upload-artifact@v3
168+
with:
169+
path: |
170+
./dist/*.whl
171+
./linux-aarch64-libchdb.tar.gz
172+
- name: Upload pypi
173+
if: startsWith(github.ref, 'refs/tags/v')
174+
run: |
175+
python3 -m pip install twine
176+
python3 -m twine upload dist/*.whl
177+
env:
178+
TWINE_USERNAME: __token__
179+
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
180+

‎.github/workflows/build_linux_arm64_wheels.yml

Lines changed: 0 additions & 115 deletions
This file was deleted.

‎contrib/jemalloc-cmake/CMakeLists.txt

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,11 @@ if (OS_LINUX)
3434
# avoid spurious latencies and additional work associated with
3535
# MADV_DONTNEED. See
3636
# https://github.com/ClickHouse/ClickHouse/issues/11121 for motivation.
37-
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:true")
37+
38+
# chdb: set background_thread true will cause Python 3.8 assert error on Linux aarch64:
39+
# <jemalloc>: contrib/jemalloc/src/jemalloc.c:4201: Failed assertion: "malloc_initialized()"
40+
# so, disable it for now.
41+
set (JEMALLOC_CONFIG_MALLOC_CONF "percpu_arena:percpu,oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000,prof:true,prof_active:false,background_thread:false")
3842
else()
3943
set (JEMALLOC_CONFIG_MALLOC_CONF "oversize_threshold:0,muzzy_decay_ms:0,dirty_decay_ms:5000")
4044
endif()

0 commit comments

Comments
 (0)