Support cross-compilation for macOS

CMakeLists.txt

@@ -353,6 +353,11 @@ set (CMAKE_ASM_FLAGS_RELWITHDEBINFO      "${CMAKE_ASM_FLAGS_RELWITHDEBINFO} -fPI
 set (CMAKE_ASM_FLAGS_DEBUG               "${CMAKE_ASM_FLAGS_DEBUG} -O${DEBUG_O_LEVEL} ${DEBUG_INFO_FLAGS} ${CMAKE_ASM_FLAGS_ADD}")
 
 if (OS_DARWIN)
+    # Set macOS deployment target if specified
+    if (CMAKE_OSX_DEPLOYMENT_TARGET)
+        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-macosx_version_min,${CMAKE_OSX_DEPLOYMENT_TARGET}")
+        set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-macosx_version_min,${CMAKE_OSX_DEPLOYMENT_TARGET}")
+    endif()
     if (USE_PYTHON)
         set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-undefined,dynamic_lookup")
     else()

chdb/build.sh

@@ -112,63 +112,6 @@ CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0
     -DCHDB_VERSION=${CHDB_VERSION} \
     "
 
-# # Generate libchdb.so linkage command:
-# #   1. Use ar to delete the LocalChdb.cpp.o from libclickhouse-local-lib.a
-# #       `ar d programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o`
-# #   2. Change the entry point from `PyInit_chdb` to `query_stable`
-# #       `-Wl,-ePyInit_chdb` to `-Wl,-equery_stable` on Linux
-# #       `-Wl,-exported_symbol,_PyInit_${CHDB_PY_MOD}` to 
-# #           `-Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result` on Darwin
-# #   3. Change the output file name from `_chdb.cpython-xx-x86_64-linux-gnu.s` to `libchdb.so`
-# #       `-o _chdb.cpython-39-x86_64-linux-gnu.so` to `-o libchdb.so`
-# #   4. Write the command to a file for debug
-# #   5. Run the command to generate libchdb.so
-
-# # Remove object from archive and save it to a new archive like:
-# # path/to/oldname.a -> path/to/oldname-nopy.a
-# remove_obj_from_archive() {
-#     local archive=$1
-#     local obj=$2
-#     local new_archive=$(echo ${archive} | sed 's/\.a$/-nopy.a/')
-#     cp -a ${archive} ${new_archive}
-#     ${AR} d ${new_archive} ${obj}
-#     echo "Old archive: ${archive}"
-#     ls -l ${archive}
-#     echo "New archive: ${new_archive}"
-#     ls -l ${new_archive}
-#     local oldfile=$(basename ${archive})
-#     local newfile=$(basename ${new_archive})
-#     LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed "s/${oldfile}/${newfile}/g")
-#     ${SED_INPLACE} "s/${oldfile}/${newfile}/g" CMakeFiles/libchdb.rsp
-# }
-
-
-# # Step 1, 2, 3:
-# #   Backup the libclickhouse-local-lib.a and restore it after ar d
-# # LIBCHDB_SO="libchdb.so"
-# # CLEAN_CHDB_A="libclickhouse-local-chdb.a"
-# # cp -a ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a.bak
-# # ${AR} d ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o
-# # mv ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a ${BUILD_DIR}/programs/local/${CLEAN_CHDB_A}
-# # mv ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a.bak ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a
-# # ls -l ${BUILD_DIR}/programs/local/
-# LIBCHDB_SO="libchdb.so"
-# LIBCHDB_CMD=${PYCHDB_CMD}
-# if [ "${build_type}" == "Debug" ]; then
-#     remove_obj_from_archive ${BUILD_DIR}/programs/local/libclickhouse-local-libd.a LocalChdb.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libdbmsd.a StoragePython.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libdbmsd.a PythonSource.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libclickhouse_common_iod.a PythonUtils.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/TableFunctions/libclickhouse_table_functionsd.a TableFunctionPython.cpp.o
-# else
-#     remove_obj_from_archive ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libdbms.a StoragePython.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libdbms.a PythonSource.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/libclickhouse_common_io.a PythonUtils.cpp.o
-#     remove_obj_from_archive ${BUILD_DIR}/src/TableFunctions/libclickhouse_table_functions.a TableFunctionPython.cpp.o
-# fi
-
-
 LIBCHDB_SO="libchdb.so"
 # Build libchdb.so
 cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 ..

chdb/build/build_static_lib_mac_on_linux.sh

@@ -0,0 +1,204 @@
+#!/bin/bash
+
+set -e
+
+# Cross-compile chdb static library for macOS (x86_64 or arm64) on Linux
+# Usage: ./build_static_lib_mac_on_linux.sh [x86_64|arm64] [Release|Debug]
+
+# Parse arguments
+TARGET_ARCH=${1:-x86_64}
+build_type=${2:-Release}
+
+# Validate architecture
+if [[ "$TARGET_ARCH" != "x86_64" && "$TARGET_ARCH" != "arm64" ]]; then
+    echo "Error: Invalid architecture. Use 'x86_64' or 'arm64'"
+    echo "Usage: $0 [x86_64|arm64] [Release|Debug]"
+    exit 1
+fi
+
+echo "Cross-compiling chdb static library for macOS ${TARGET_ARCH} on Linux..."
+
+# Verify we're running on Linux
+if [ "$(uname)" != "Linux" ]; then
+    echo "Error: This script must be run on Linux"
+    exit 1
+fi
+
+# Verify required environment variables
+if [ -z "${CCTOOLS:-}" ]; then
+    echo "Error: CCTOOLS environment variable not set. Please set it to the cctools bin directory."
+    echo "Example: export CCTOOLS=/path/to/cctools"
+    exit 1
+fi
+
+# Set architecture-specific variables
+if [ "$TARGET_ARCH" == "x86_64" ]; then
+    DARWIN_TRIPLE="x86_64-apple-darwin"
+    CMAKE_ARCH="x86_64"
+    TOOLCHAIN_FILE="cmake/darwin/toolchain-x86_64.cmake"
+    BUILD_DIR_SUFFIX="static-lib-darwin-x86_64"
+    OUTPUT_SUFFIX="darwin-x86_64"
+    EXAMPLE_DIR_SUFFIX="darwin-x86_64"
+    MACOS_MIN_VERSION="10.15"
+    # x86_64 specific: disable AVX for compatibility
+    CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+else
+    # arm64
+    DARWIN_TRIPLE="aarch64-apple-darwin"
+    CMAKE_ARCH="aarch64"
+    TOOLCHAIN_FILE="cmake/darwin/toolchain-aarch64.cmake"
+    BUILD_DIR_SUFFIX="static-lib-darwin-arm64"
+    OUTPUT_SUFFIX="darwin-arm64"
+    EXAMPLE_DIR_SUFFIX="darwin-arm64"
+    MACOS_MIN_VERSION="11.0"
+    # ARM64 specific: disable x86 features
+    CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=0"
+fi
+
+# Check if cctools exist for this architecture
+if [ ! -f "${CCTOOLS}/bin/${DARWIN_TRIPLE}-ar" ]; then
+    echo "Error: cctools not found at ${CCTOOLS}/bin/${DARWIN_TRIPLE}-ar"
+    echo "Tip: You may need to rebuild cctools with support for ${TARGET_ARCH}"
+    exit 1
+fi
+
+MY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+. ${MY_DIR}/../vars.sh
+
+BUILD_DIR=${PROJ_DIR}/build-${BUILD_DIR_SUFFIX}
+
+# Set up cross-compilation tools
+export CC=clang-19
+export CXX=clang++-19
+
+# macOS-specific settings
+GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0"
+UNWIND="-DUSE_UNWIND=0"
+HDFS="-DENABLE_HDFS=0 -DENABLE_GSASL_LIBRARY=0 -DENABLE_KRB5=0"
+MYSQL="-DENABLE_MYSQL=0"
+ICU="-DENABLE_ICU=0"
+RUST_FEATURES="-DENABLE_RUST=0"
+JEMALLOC="-DENABLE_JEMALLOC=0"
+LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
+
+if [ ! -d $BUILD_DIR ]; then
+    mkdir $BUILD_DIR
+fi
+
+cd ${BUILD_DIR}
+
+CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \
+    -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \
+    -DENABLE_AZURE_BLOB_STORAGE=1 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \
+    -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \
+    -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \
+    -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \
+    -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \
+    -DENABLE_LDAP=0 \
+    ${MYSQL} \
+    ${HDFS} \
+    -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \
+    ${GLIBC_COMPATIBILITY} \
+    -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \
+    ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \
+    -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \
+    -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 -DENABLE_MSGPACK=1 \
+    -DENABLE_BROTLI=1 -DENABLE_H3=1 -DENABLE_CURL=1 \
+    -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \
+    -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \
+    ${CPU_FEATURES} \
+    -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \
+    -DENABLE_LIBFIU=1 \
+    -DCHDB_VERSION=${CHDB_VERSION} \
+    -DCMAKE_AR:FILEPATH=${CCTOOLS}/bin/${DARWIN_TRIPLE}-ar \
+    -DCMAKE_INSTALL_NAME_TOOL=${CCTOOLS}/bin/${DARWIN_TRIPLE}-install_name_tool \
+    -DCMAKE_RANLIB:FILEPATH=${CCTOOLS}/bin/${DARWIN_TRIPLE}-ranlib \
+    -DCMAKE_LINKER:FILEPATH=${CCTOOLS}/bin/${DARWIN_TRIPLE}-ld \
+    -DLINKER_NAME=${CCTOOLS}/bin/${DARWIN_TRIPLE}-ld \
+    -DCMAKE_TOOLCHAIN_FILE=${TOOLCHAIN_FILE} \
+    "
+
+echo "Running cmake configuration..."
+cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 -DCHDB_STATIC_LIBRARY_BUILD=1 ..
+
+echo "Building with ninja..."
+ninja -d keeprsp
+
+BINARY=${BUILD_DIR}/programs/clickhouse
+rm -f ${BINARY}
+
+cd ${BUILD_DIR}
+ninja -d keeprsp -v > build.log || true
+
+ccache -s || true
+
+cd ${MY_DIR}
+
+# Create static library
+echo "Creating static library libchdb.a for macOS..."
+python3 create_static_libchdb.py
+if [ $? -ne 0 ]; then
+    echo "Error: Failed to create static library"
+    exit 1
+fi
+
+# Prepare cpp-example directory and copy header file
+echo "Preparing cpp-example-${EXAMPLE_DIR_SUFFIX} directory..."
+if [ ! -d ${MY_DIR}/cpp-example-${EXAMPLE_DIR_SUFFIX} ]; then
+    cp -r ${MY_DIR}/cpp-example ${MY_DIR}/cpp-example-${EXAMPLE_DIR_SUFFIX}
+fi
+
+cd ${MY_DIR}/cpp-example-${EXAMPLE_DIR_SUFFIX}
+cp ${PROJ_DIR}/programs/local/chdb.h .
+cp ${MY_DIR}/libchdb.a .
+echo "Copied chdb.h and libchdb.a to cpp-example-${EXAMPLE_DIR_SUFFIX} directory"
+
+echo "Note: Skipping C++ example compilation for cross-compilation."
+echo "The example can be compiled on the target macOS ${TARGET_ARCH} system with:"
+echo "  clang chdb_example.cpp -o chdb_example -mmacosx-version-min=${MACOS_MIN_VERSION} -L. -lchdb -liconv -framework CoreFoundation"
+
+# For cross-compilation, we'll create a minimal analysis without running the compiled binary
+echo "Creating analysis files for cross-compilation..."
+
+# Copy map file analysis tools but don't run them (since we can't execute macOS binaries on Linux)
+echo "Note: Skipping map file analysis for cross-compilation."
+echo "Run the following on macOS ${TARGET_ARCH} to create minimal library:"
+echo "  cd ${MY_DIR}/cpp-example-${EXAMPLE_DIR_SUFFIX}"
+echo "  clang chdb_example.cpp -o chdb_example -mmacosx-version-min=${MACOS_MIN_VERSION} -L. -lchdb -liconv -framework CoreFoundation -Wl,-map,chdb_example.map"
+echo "  cd ${MY_DIR}"
+echo "  python3 extract_chdb_objects.py --map-file=cpp-example-${EXAMPLE_DIR_SUFFIX}/chdb_example.map"
+echo "  python3 create_minimal_libchdb.py"
+
+# For now, we'll use the full libchdb.a as the final output
+echo "Using full libchdb.a for cross-compilation (minimal version requires macOS execution)"
+
+# Strip the libchdb.a if not debug build
+if [ ${build_type} == "Debug" ]; then
+    echo -e "\nDebug build, skip strip"
+else
+    echo -e "\nStrip the libchdb.a:"
+    # Use macOS-compatible strip command via cctools
+    if [ -f "${CCTOOLS}/bin/${DARWIN_TRIPLE}-strip" ]; then
+        ${CCTOOLS}/bin/${DARWIN_TRIPLE}-strip -S -x libchdb.a
+    else
+        echo "Warning: macOS strip not found, skipping strip step"
+    fi
+fi
+
+echo "Note: Skipping Go test for cross-compilation."
+
+# Copy final library to project root
+OUTPUT_NAME="libchdb-${OUTPUT_SUFFIX}.a"
+echo "Copying libchdb.a to project root as ${OUTPUT_NAME}..."
+cp ${MY_DIR}/libchdb.a ${PROJ_DIR}/${OUTPUT_NAME}
+echo "Final ${OUTPUT_NAME} created at ${PROJ_DIR}/${OUTPUT_NAME}"
+
+# Print final library size
+echo "Final ${OUTPUT_NAME} size:"
+ls -lh ${PROJ_DIR}/${OUTPUT_NAME}
+
+echo "Cross-compilation for macOS ${TARGET_ARCH} completed successfully!"
+echo "Generated files:"
+echo "  - ${PROJ_DIR}/${OUTPUT_NAME}"
+echo "  - ${MY_DIR}/cpp-example-${EXAMPLE_DIR_SUFFIX}/ (for testing on macOS ${TARGET_ARCH})"

chdb/build/download_python_headers.sh

@@ -0,0 +1,113 @@
+#!/bin/bash
+
+set -e
+
+TARGET_DIR="${HOME}/python_include"
+TEMP_DIR="${TARGET_DIR}/tmp"
+
+VERSIONS=(
+    "3.8.10:3.8:3.8"
+    "3.9.13:3.9:3.9"
+    "3.10.11:3.10:3.10"
+    "3.11.9:3.11:3.11"
+    "3.12.10:3.12:3.12"
+    "3.13.9:3.13:3.13"
+    "3.14.0:3.14:3.14"
+)
+
+cleanup() {
+    rm -rf "$TEMP_DIR"
+}
+trap cleanup EXIT
+
+mkdir -p "$TARGET_DIR"
+mkdir -p "$TEMP_DIR"
+
+for entry in "${VERSIONS[@]}"; do
+    IFS=':' read -r FULL_VER SUBDIR MINOR_VER <<< "$entry"
+
+    echo "=========================================="
+    echo "Processing Python ${FULL_VER}..."
+    echo "=========================================="
+
+    # 检查目标目录是否已存在
+    DEST_DIR="${TARGET_DIR}/${SUBDIR}"
+    if [ -d "$DEST_DIR" ] && [ -f "${DEST_DIR}/Python.h" ]; then
+        echo "✓ Python ${FULL_VER} headers already installed at ${DEST_DIR}"
+        echo "  Skipping..."
+        continue
+    fi
+
+    WORK_DIR="${TEMP_DIR}/${SUBDIR}"
+    mkdir -p "$WORK_DIR"
+    cd "$WORK_DIR"
+
+    PKG_URL="https://www.python.org/ftp/python/${FULL_VER}/python-${FULL_VER}-macos11.pkg"
+
+    echo "Downloading: $PKG_URL"
+    if wget -q --spider "$PKG_URL" 2>/dev/null; then
+        wget -q --show-progress -O python.pkg "$PKG_URL"
+    else
+        echo "ERROR: Failed to download Python ${FULL_VER}"
+        exit 1
+    fi
+
+    echo "Extracting pkg with 7z..."
+    7z x -y python.pkg > /dev/null
+
+    PAYLOAD_DIR=""
+    for dir in Python_Framework.pkg PythonFramework-*.pkg; do
+        if [ -d "$dir" ] || [ -f "$dir/Payload" ]; then
+            PAYLOAD_DIR="$dir"
+            break
+        fi
+    done
+
+    if [ -z "$PAYLOAD_DIR" ]; then
+        PAYLOAD_DIR=$(find . -name "Payload" -type f | head -1 | xargs dirname)
+    fi
+
+    if [ -z "$PAYLOAD_DIR" ] || [ ! -f "${PAYLOAD_DIR}/Payload" ]; then
+        echo "ERROR: Cannot find Payload for Python ${FULL_VER}"
+        exit 1
+    fi
+
+    echo "Extracting Payload from ${PAYLOAD_DIR}..."
+    cd "$PAYLOAD_DIR"
+    7z x -y Payload -so 2>/dev/null | cpio -id 2>/dev/null || true
+
+    HEADER_SRC=""
+    for path in \
+        "Versions/${MINOR_VER}/Headers" \
+        "Headers"
+    do
+        if [ -d "$path" ] && [ -f "$path/Python.h" ]; then
+            HEADER_SRC="$path"
+            break
+        fi
+    done
+
+    if [ -z "$HEADER_SRC" ]; then
+        PYTHON_H=$(find . -name "Python.h" -type f | head -1)
+        if [ -n "$PYTHON_H" ]; then
+            HEADER_SRC=$(dirname "$PYTHON_H")
+        fi
+    fi
+
+    if [ -z "$HEADER_SRC" ] || [ ! -f "${HEADER_SRC}/Python.h" ]; then
+        echo "ERROR: Cannot find headers for Python ${FULL_VER}"
+        exit 1
+    fi
+
+    mkdir -p "$DEST_DIR"
+    cp -r "${HEADER_SRC}/"* "$DEST_DIR/"
+
+    echo "✓ Python ${FULL_VER} headers installed to ${DEST_DIR}"
+    echo "  Files: $(ls "$DEST_DIR" | wc -l | tr -d ' ') items"
+done
+
+echo ""
+echo "=========================================="
+echo "Done! Headers installed to: ${TARGET_DIR}"
+echo "=========================================="
+ls -la "$TARGET_DIR"