diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 60337ad..2268dff 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -79,6 +79,7 @@ jobs:
             archs: "aarch64"
 
     env:
+      PIP_ONLY_BINARY: "scipy,numpy"
       CIBW_SKIP: cp36-* cp37-* cp38-* cp39-* pp* *-musllinux_aarch64 cp312-*_i686 cp313-*
       CIBW_ARCHS: ${{matrix.archs}}
       CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"