Enable MacOS support (#14)

Add arm support. Add tests to deployment.
dipdup-io · Aug 21, 2024 · 16128b6 · 16128b6
1 parent 4c2ff5b
commit 16128b6
Show file tree

Hide file tree

Showing 17 changed files with 667 additions and 121 deletions.
diff --git a/.github/workflows/ghcr.yml b/.github/workflows/ghcr.yml
@@ -12,15 +12,15 @@ on:
 jobs:
   build:
     name: Docker images for ghcr.io
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     env:
       ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       DOCKER_REGISTRY: ghcr.io
       DOCKER_IMAGE_BASE: ${{ github.repository }}
     steps:
       - name: Check out the repo
         uses: actions/checkout@v3
-
+      
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
 
@@ -48,8 +48,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta-all.outputs.tags }}
-          labels: ${{ steps.meta-all.coutputs.labels }}
-
+          labels: ${{ steps.meta-all.outputs.labels }}
 
       # Prover
 
@@ -58,6 +57,7 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_BASE }}/cpu_air_prover
+
       - name: Prover image build & push
         uses: docker/build-push-action@v3
         with:
@@ -67,7 +67,7 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.coutputs.labels }}
+          labels: ${{ steps.meta.outputs.labels }}
 
       # Verifier
 
@@ -76,6 +76,7 @@ jobs:
         uses: docker/metadata-action@v3
         with:
           images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_BASE }}/cpu_air_verifier
+
       - name: Verifier image build & push
         uses: docker/build-push-action@v3
         with:
@@ -85,4 +86,4 @@ jobs:
           cache-from: type=gha
           cache-to: type=gha,mode=max
           tags: ${{ steps.meta-ver.outputs.tags }}
-          labels: ${{ steps.meta-ver.coutputs.labels }}
+          labels: ${{ steps.meta-ver.outputs.labels }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -7,47 +7,69 @@ on:
 
 jobs:
   build:
-    name: Docker images for ghcr.io
-    runs-on: ubuntu-latest
-    env:
-      ACTIONS_ALLOW_UNSECURE_COMMANDS: true
-      DOCKER_REGISTRY: ghcr.io
-      DOCKER_IMAGE_BASE: ${{ github.repository }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-22.04, macos-14]
+
     steps:
-      - name: Check out the repo
-        uses: actions/checkout@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Log in to the registry
-        uses: docker/login-action@v2
-        with:
-          registry: ${{ env.DOCKER_REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Prover and Verifier and deb
-      - name: Set output
-        id: vars
-        run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
-
-      - name: Extract files and build DEB package
-        run: |
-          chmod +x ./build_deb.sh  # Ensure your script is executable
-          ./build_deb.sh ${{ steps.vars.outputs.tag }}
-
-      - name: Verify file existence#2
-        run: ls -la /tmp/stone-prover
-
-      - name: Upload files to a GitHub release
-        id: create_release
-        uses: softprops/action-gh-release@v2
-        with:
-          files: /tmp/stone-prover/usr/bin/cpu_air*
-
-      - name: Upload dep to a GitHub release
-        id: create_release_deb
-        uses: softprops/action-gh-release@v2
-        with:
-          files: /tmp/stone-prover/stone-prover.deb
+    - name: Checkout code
+      uses: actions/checkout@v2
+
+    - name: Set up environment
+      run: |
+        echo "Setting up environment for ${{ matrix.os }}"
+        if [ "${{ matrix.os }}" == "ubuntu-22.04" ]; then
+          echo "TARGET_ARCH=x86_64" >> $GITHUB_ENV
+        elif [ "${{ matrix.os }}" == "macos-14" ]; then
+          echo "TARGET_ARCH=arm64" >> $GITHUB_ENV
+        fi
+
+    - name: Build
+      run: |
+        chmod +x ./build.sh
+        ./build.sh
+    
+    - name: Test
+      run: |
+        chmod +x ./test.sh
+        ./test.sh
+
+    - name: Set output
+      id: vars
+      run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT
+
+    - name: Extract files and build DEB package
+      if: matrix.os == 'ubuntu-22.04'
+      run: |
+        chmod +x ./package_deb.sh
+        ./package_deb.sh ${{ steps.vars.outputs.tag }}
+
+    - name: Rename binaries ubuntu
+      if: matrix.os == 'ubuntu-22.04'
+      run: |
+        echo "Renaming binaries to include architecture"
+        mv /tmp/stone-prover/build/bazelbin/src/starkware/main/cpu/cpu_air_prover ./cpu_air_prover-${TARGET_ARCH}
+        mv /tmp/stone-prover/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier ./cpu_air_verifier-${TARGET_ARCH}
+        mv /tmp/stone-prover/stone-prover.deb /tmp/stone-prover/stone-prover-linux-${TARGET_ARCH}.deb
+
+    - name: Rename binaries macos
+      if: matrix.os == 'macos-14'
+      run: |
+        echo "Renaming binaries to include architecture"
+        mv /tmp/stone-prover/build/bazelbin/src/starkware/main/cpu/cpu_air_prover ./cpu_air_prover-${TARGET_ARCH}
+        mv /tmp/stone-prover/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier ./cpu_air_verifier-${TARGET_ARCH}
+
+    - name: Verify file existence#2
+      run: |
+        ls -la ./
+        ls -la /tmp/stone-prover/
+
+    - name: Upload files to a GitHub release
+      uses: softprops/action-gh-release@v2
+      with:
+        files: |
+         ./cpu_air*
+         /tmp/stone-prover/*.deb
+
diff --git a/Dockerfile b/Dockerfile
@@ -1,32 +1,59 @@
 # Stage 1: Base Image
-FROM ciimage/python:3.9 AS base_image
+FROM --platform=$BUILDPLATFORM ubuntu:22.04 AS build
 
-# Install necessary dependencies including git
 RUN apt-get update && apt-get install -y git
 
-# Clone the public prover repository
-RUN git clone https://github.com/starkware-libs/stone-prover.git /app/prover
+RUN git clone https://github.com/baking-bad/stone-prover.git /app
 
-# Set the working directory to the cloned repository
-WORKDIR /app/prover
+WORKDIR /app
 
-# Run the installation scripts from the cloned repository
-RUN /app/prover/install_deps.sh
-RUN ./docker_common_deps.sh
+COPY . .
 
-# Change ownership of the /app directory
-RUN chown -R starkware:starkware /app
+# Install dependencies.
+RUN ./install_deps.sh
 
-# Build the project using Bazel
-RUN bazel build //...
+# Build.
+RUN bazelisk build //...
+
+FROM build AS test
+
+# Run tests.
+RUN bazelisk test //...
+
+# Copy cpu_air_prover and cpu_air_verifier.
+RUN ln -s /app/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /bin/cpu_air_prover
+RUN ln -s /app/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier /bin/cpu_air_verifier
+
+# End to end test.
+WORKDIR /app/e2e_test/CairoZero
+
+RUN cairo-compile fibonacci.cairo --output fibonacci_compiled.json --proof_mode
+
+RUN cairo-run \
+    --program=fibonacci_compiled.json \
+    --layout=small \
+    --program_input=fibonacci_input.json \
+    --air_public_input=fibonacci_public_input.json \
+    --air_private_input=fibonacci_private_input.json \
+    --trace_file=fibonacci_trace.json \
+    --memory_file=fibonacci_memory.json \
+    --print_output \
+    --proof_mode
+
+RUN cpu_air_prover \
+    --out_file=fibonacci_proof.json \
+    --private_input_file=fibonacci_private_input.json \
+    --public_input_file=fibonacci_public_input.json \
+    --prover_config_file=cpu_air_prover_config.json \
+    --parameter_file=cpu_air_params.json
+
+RUN cpu_air_verifier --in_file=fibonacci_proof.json && echo "Successfully verified example proof."
 
 # Stage 2: Target Image
 FROM debian:stable-slim AS target
 
-# Copy the built binary from the base image to the target image
-COPY --from=base_image /app/prover/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /usr/bin/
-# Uncomment the following line if you need to copy the verifier as well
-COPY --from=base_image /app/prover/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier /usr/bin/
+COPY --from=build /app/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /usr/bin/
+COPY --from=build /app/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier /usr/bin/
 
 # Install the necessary runtime dependencies
 RUN apt update && apt install -y libdw1
diff --git a/README.md b/README.md
@@ -9,24 +9,34 @@ The goal of this project is to reduce the friction and time to start producing p
 ## Roadmap
 
 - [x] Releases with static binaries for x86_64
+- [x] Releases with static binaries for arm64
 - [x] Minimal docker images for x86_64
 - [x] Native packages for Debian/Ubuntu
-- [ ] Native packages for Fedora
-- [ ] ARM builds
 
 Follow-up work:
+- Native packages for Fedora
+- Native packages for Alpine
+- Native packages for Homebrew
 - Technical docs for file formats (inputs, outputs, memory, trace, proof), test data
+- Docs hosted on github pages
 - Integrated proof decomposition (related to https://github.com/zksecurity/stark-evm-adapter)
 - Observability suite (metrics, dashboard, configurable logging)
 - Stwo support
 
 # Instructions for use
 
-## Download binaries for x86_64
+### Download binaries for x86_64
 
 ```bash
-sudo wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_prover -O /usr/bin/cpu_air_prover && sudo chmod +x /usr/bin/cpu_air_prover
-sudo wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_verifier -O /usr/bin/cpu_air_verifier && sudo chmod +x /usr/bin/cpu_air_verifier
+sudo wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_prover-x86_64 -O /usr/local/bin/cpu_air_prover && sudo chmod +x /usr/local/bin/cpu_air_prover
+sudo wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_verifier-x86_64 -O /usr/local/bin/cpu_air_verifier && sudo chmod +x /usr/local/bin/cpu_air_verifier
+```
+
+### Download binaries for MacOS Arm64
+
+```bash
+wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_prover-arm64 -O /usr/local/bin/cpu_air_prover && chmod +x /usr/local/bin/cpu_air_prover
+wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/cpu_air_verifier-arm64 -O /usr/local/bin/cpu_air_verifier && chmod +x /usr/local/bin/cpu_air_verifier
 ```
 
 ### Creating and verifying a test proof using binaries
@@ -78,12 +88,6 @@ Clone the repository:
 git clone https://github.com/dipdup-io/stone-packaging.git /tmp/stone-packaging
 ```
 
-Navigate to the example test directory (`/tmp/stone-packaging/test_files/`):
-
-```bash
-cd /tmp/stone-packaging/test_files/
-```
-
 Run docker images with volume:
 
 ```bash
@@ -108,7 +112,7 @@ docker run --entrypoint /bin/bash -v /tmp/stone-packaging/test_files:/app/prover
 Download the deb package from the latest release. For example:
 
 ```bash
-wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/stone-prover.deb && sudo dpkg -i stone-prover.deb
+wget https://github.com/dipdup-io/stone-packaging/releases/latest/download/stone-prover-linux-x86_64.deb && sudo dpkg -i stone-prover-linux-x86_64.deb
 ```
 
 ### Creating and verifying a test proof using deb package

diff --git a/air_prover/Dockerfile b/air_prover/Dockerfile
@@ -1,35 +1,58 @@
 # Stage 1: Base Image
-FROM ciimage/python:3.9 AS base_image
+FROM ubuntu:22.04 AS build
 
-# Install necessary dependencies including git
 RUN apt-get update && apt-get install -y git
 
-# Clone the public prover repository
-RUN git clone https://github.com/starkware-libs/stone-prover.git /app/prover
+RUN git clone https://github.com/baking-bad/stone-prover.git /app
 
-# Set the working directory to the cloned repository
-WORKDIR /app/prover
+WORKDIR /app
 
-# Run the installation scripts from the cloned repository
-RUN /app/prover/install_deps.sh
-RUN ./docker_common_deps.sh
+COPY . .
 
-# Change ownership of the /app directory
-RUN chown -R starkware:starkware /app
+# Install dependencies.
+RUN ./install_deps.sh
 
-# Build the project using Bazel
-RUN bazel build //...
+# Build.
+RUN bazelisk build //...
+
+FROM build AS test
+
+# Run tests.
+RUN bazelisk test //...
+
+# Copy cpu_air_prover and cpu_air_verifier.
+RUN ln -s /app/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /bin/cpu_air_prover
+RUN ln -s /app/build/bazelbin/src/starkware/main/cpu/cpu_air_verifier /bin/cpu_air_verifier
+
+# End to end test.
+WORKDIR /app/e2e_test/CairoZero
+
+RUN cairo-compile fibonacci.cairo --output fibonacci_compiled.json --proof_mode
+
+RUN cairo-run \
+    --program=fibonacci_compiled.json \
+    --layout=small \
+    --program_input=fibonacci_input.json \
+    --air_public_input=fibonacci_public_input.json \
+    --air_private_input=fibonacci_private_input.json \
+    --trace_file=fibonacci_trace.json \
+    --memory_file=fibonacci_memory.json \
+    --print_output \
+    --proof_mode
+
+RUN cpu_air_prover \
+    --out_file=fibonacci_proof.json \
+    --private_input_file=fibonacci_private_input.json \
+    --public_input_file=fibonacci_public_input.json \
+    --prover_config_file=cpu_air_prover_config.json \
+    --parameter_file=cpu_air_params.json
+
+RUN cpu_air_verifier --in_file=fibonacci_proof.json && echo "Successfully verified example proof."
 
 # Stage 2: Target Image
 FROM debian:stable-slim AS target
 
-# Copy the built binary from the base image to the target image
-COPY --from=base_image /app/prover/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /usr/bin/
-# Uncomment the following line if you need to copy the verifier as well
-# COPY --from=base_image /app/prover/bazel-bin/src/starkware/main/cpu/cpu_air_verifier /usr/bin/
+COPY --from=build /app/build/bazelbin/src/starkware/main/cpu/cpu_air_prover /usr/bin/
 
-# Install the necessary runtime dependencies
 RUN apt update && apt install -y libdw1
-
-# Set the entry point for the container
 ENTRYPOINT ["cpu_air_prover"]