Add support for querying Python Arrow tables directly

.github/workflows/build_linux_arm64_wheels-gh.yml

@@ -138,6 +138,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Check ccache statistics
         run: |
           ccache -s

.github/workflows/build_linux_x86_wheels.yml

@@ -138,6 +138,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Check ccache statistics
         run: |
           ccache -s

.github/workflows/build_macos_arm64_wheels.yml

@@ -22,8 +22,21 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (macOS ARM64)
-    runs-on: macos-13-xlarge
+    runs-on: macos-14-xlarge
     steps:
+      - name: Check machine architecture
+        run: |
+          echo "=== Machine Architecture Information ==="
+          echo "Machine type: $(uname -m)"
+          echo "Architecture: $(arch)"
+          echo "System info: $(uname -a)"
+          echo "Hardware info:"
+          system_profiler SPHardwareDataType | grep "Chip\|Processor"
+          if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then
+            echo "This is an ARM64 (Apple Silicon) machine"
+          else
+            echo "This is an x86_64 (Intel) machine"
+          fi
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -79,7 +92,7 @@ jobs:
           brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
           brew install --ignore-dependencies llvm@19
           brew install git ninja libtool gettext gcc binutils grep findutils nasm
-          brew install --build-from-source ccache
+          brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
           export PATH=$(brew --prefix llvm@19)/bin:$PATH
@@ -97,7 +110,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
-          key: macos-13-xlarge
+          key: macos-14-xlarge
           max-size: 5G
           append-timestamp: true
       - name: Run chdb/build.sh
@@ -141,6 +154,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Keep killall ccache and wait for ccache to finish
         if: always()
         run: |

.github/workflows/build_macos_x86_wheels.yml

@@ -22,8 +22,21 @@ on:
 jobs:
   build_universal_wheel:
     name: Build Universal Wheel (macOS x86_64)
-    runs-on: macos-13
+    runs-on: macos-14-large
     steps:
+      - name: Check machine architecture
+        run: |
+          echo "=== Machine Architecture Information ==="
+          echo "Machine type: $(uname -m)"
+          echo "Architecture: $(arch)"
+          echo "System info: $(uname -a)"
+          echo "Hardware info:"
+          system_profiler SPHardwareDataType | grep "Chip\|Processor"
+          if sysctl -n hw.optional.arm64 2>/dev/null | grep -q "1"; then
+            echo "This is an ARM64 (Apple Silicon) machine"
+          else
+            echo "This is an x86_64 (Intel) machine"
+          fi
       - name: Setup pyenv
         run: |
           curl https://pyenv.run | bash
@@ -79,7 +92,7 @@ jobs:
           brew install ca-certificates lz4 mpdecimal openssl@3 readline sqlite xz z3 zstd
           brew install --ignore-dependencies llvm@19
           brew install git ninja libtool gettext gcc binutils grep findutils nasm
-          brew install --build-from-source ccache
+          brew install ccache || echo "ccache installation failed, continuing without it"
           brew install go
           cd /usr/local/opt/ && sudo rm -f llvm && sudo ln -sf llvm@19 llvm
           export PATH=$(brew --prefix llvm@19)/bin:$PATH
@@ -97,7 +110,7 @@ jobs:
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1.2
         with:
-          key: macos-13-x86_64
+          key: macos-14-x86_64
           max-size: 5G
           append-timestamp: true
       - name: Run chdb/build.sh
@@ -142,6 +155,7 @@ jobs:
       - name: Run libchdb stub in examples dir
         run: |
           bash -x ./examples/runStub.sh
+          bash -x ./examples/runArrowTest.sh
       - name: Keep killall ccache and wait for ccache to finish
         if: always()
         run: |

README.md

@@ -416,11 +416,7 @@ chDB automatically converts Python dictionary objects to ClickHouse JSON types f
       ```
     - Columns are converted to `String` if sampling finds non-dictionary values.
 
-2. **Arrow Table**
-    - `struct` type columns are automatically mapped to JSON columns.
-    - Nested structures preserve type information.
-
-3. **chdb.PyReader**
+2. **chdb.PyReader**
     - Implement custom schema mapping in `get_schema()`:
       ```python
       def get_schema(self):

chdb/__init__.py

@@ -103,7 +103,9 @@ def to_arrowTable(res):
         raise ImportError("Failed to import pyarrow or pandas") from None
     if len(res) == 0:
         return pa.Table.from_batches([], schema=pa.schema([]))
-    return pa.RecordBatchFileReader(res.bytes()).read_all()
+
+    memview = res.get_memview()
+    return pa.RecordBatchFileReader(memview.view()).read_all()
 
 
 # return pandas dataframe

chdb/state/sqlitelike.py

@@ -62,7 +62,9 @@ def to_arrowTable(res):
         raise ImportError("Failed to import pyarrow or pandas") from None
     if len(res) == 0:
         return pa.Table.from_batches([], schema=pa.schema([]))
-    return pa.RecordBatchFileReader(res.bytes()).read_all()
+
+    memview = res.get_memview()
+    return pa.RecordBatchFileReader(memview.view()).read_all()
 
 
 # return pandas dataframe