feat: merkle root ffi

.devcontainer/Dockerfile

@@ -1,8 +1,30 @@
 FROM python
+
+# Configure Poetry
 ENV PYTHONPATH=.
+ENV POETRY_VERSION=1.3.2
+ENV POETRY_HOME=/opt/poetry
+ENV POETRY_VENV=/opt/poetry-venv
+ENV POETRY_CACHE_DIR=/opt/.cache
+
+# Install poetry separated from system interpreter
+RUN python3 -m venv $POETRY_VENV && $POETRY_VENV/bin/pip install -U pip setuptools && $POETRY_VENV/bin/pip install poetry==${POETRY_VERSION}
+
+# Add `poetry` to PATH
+ENV PATH="${PATH}:${POETRY_VENV}/bin"
 
 # install deps
-RUN apt update -y && apt install curl zsh git -y
-# poetry, ohMyZsh
-RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python -
+RUN apt update -y && apt upgrade
+RUN apt install -y curl zsh git build-essential gnupg && \
+    curl -sS https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - && \
+    echo "deb https://dl.yarnpkg.com/debian/ stable main" | tee /etc/apt/sources.list.d/yarn.list
+
+# Instalar o Yarn
+RUN apt-get update && apt-get install -y yarn
+
+# Intall Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain nightly
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# ohMyZsh
 RUN sh -c "$(curl -fsSL https://raw.github.com/ohmyzsh/ohmyzsh/master/tools/install.sh)"

.devcontainer/devcontainer.json

@@ -1,16 +1,12 @@
 {
-	"name": "Existing Dockerfile",
-	"context": "..",
-	"dockerFile": "./Dockerfile",
-	"postCreateCommand": "poetry install && pip install --upgrade pip",
-	"settings": {
-		"python.pythonPath": "."
-	},
-	"extensions": [
-		"bungcip.better-toml",
-		"mhutchie.git-graph",
-		"Tyriar.sort-lines",
-		"ms-python.python",
-		"eamodio.gitlens"
-	]
-}
+  "name": "Existing Dockerfile",
+  "context": "..",
+  "dockerFile": "./Dockerfile",
+  "extensions": [
+    "bungcip.better-toml",
+    "mhutchie.git-graph",
+    "Tyriar.sort-lines",
+    "ms-python.python",
+    "eamodio.gitlens"
+  ]
+}

.github/workflows/ci.yml

@@ -23,24 +23,35 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v3
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
           pip install pytest
           pip install -e .
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
       - name: Set up Node.js
         uses: actions/setup-node@v2
         with:
           node-version: "18"
+
+      - name: Set up Rust Minimal Nightly
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+
       - name: Install Node.js dependencies
         run: |
           cd test/merkletreejs
           yarn install
+
       - name: Test with pytest
         run: |
-          pytest -m "not benchmark" -vv
+          pytest -m "not benchmark" -vv

.gitignore

@@ -1,4 +1,6 @@
 node_modules
+target
+
 .hypothesis
 .vscode
 *cache*

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ repos:
     hooks:
       - id: pytest
         name: Unit Tests (Pytest)
-        entry: pytest -m "not benchmark and not merkletreejs"
+        entry: pytest -m "not benchmark"
         language: system
         pass_filenames: false
         always_run: true

merkly/accelerator/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "accelerator"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tiny-keccak = { version = "2.0.2", features = ["keccak"] }
+
+
+[lib]
+crate-type = ["cdylib"]
+
+[profile.release]
+strip = true

merkly/accelerator/build.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+cargo b -r --out-dir . -Z unstable-options

merkly/accelerator/mtreers.py

@@ -0,0 +1,93 @@
+from ctypes import CDLL, POINTER, Structure, c_size_t, c_ubyte
+from merkly.node import Node
+from typing import List
+
+
+class MerkleProof(Structure):
+    _fields_ = [("ptr", POINTER(POINTER(c_ubyte))), ("len", c_size_t)]
+
+
+class MTreers:
+    def __init__(self) -> None:
+        self.lib = CDLL("./merkly/accelerator/libaccelerator.dylib")
+
+        self.lib.make_root.argtypes = [POINTER(POINTER(c_ubyte)), c_size_t]
+        self.lib.make_root.restype = POINTER(c_ubyte)
+
+        self.lib.free_root.argtypes = [POINTER(c_ubyte)]
+        self.lib.free_root.restype = None
+
+        self.lib.make_proof.argtypes = [
+            POINTER(POINTER(c_ubyte)),
+            c_size_t,
+            POINTER(c_ubyte),
+        ]
+        self.lib.make_proof.restype = MerkleProof
+
+        self.lib.free_proof.argtypes = [POINTER(POINTER(c_ubyte))]
+        self.lib.free_proof.restype = None
+
+        # self.lib.verify.argtypes = [
+        #     POINTER(POINTER(c_ubyte)),
+        #     c_size_t,
+        #     POINTER(c_ubyte),
+        # ]
+        # self.lib.verify.restype = c_bool
+
+    def make_root(self, leaves: List[bytes]) -> bytes:
+        len_leaves = len(leaves)
+        leaves_pointers = (POINTER(c_ubyte) * len_leaves)()
+
+        for i, leaf in enumerate(leaves):
+            array_type = c_ubyte * 32
+            leaves_pointers[i] = array_type(*leaf)
+
+        root_ptr = self.lib.make_root(leaves_pointers, len_leaves)
+        root = bytes(root_ptr[:32])
+        self.lib.free_root(root_ptr)
+        return root
+
+    def make_proof(self, leaves: List[bytes], leaf: bytes) -> bytes:
+        leaf_pointer = (c_ubyte * 32)(*leaf)
+        len_leaves = len(leaves)
+        leaves_pointers = (POINTER(c_ubyte) * len_leaves)()
+
+        for i, leaf in enumerate(leaves):
+            array_type = c_ubyte * 32
+            leaves_pointers[i] = array_type(*leaf)
+
+        result_struct: MerkleProof = self.lib.make_proof(
+            leaves_pointers, len_leaves, leaf_pointer
+        )
+
+        proof = []
+        for i in range(result_struct.len):
+            node_ptr = result_struct.ptr[i]
+            node_data = bytes(node_ptr[:33])
+
+            flag = node_data[32]
+            data = node_data[:32]
+
+            proof.append(Node(data=data, side=flag))
+
+        self.lib.free_proof(result_struct.ptr, result_struct.len)
+        return proof
+
+    def verify(self, proof: List[Node], leaf: bytes) -> bytes:
+        proof_bytes = []
+
+        for node in proof:
+            flag = node.side.value
+            data = node.data
+            proof_bytes.append([*data, flag])
+
+        len_proof = len(proof_bytes)
+        proof_pointers = (POINTER(c_ubyte) * len_proof)()
+
+        for i, node in enumerate(proof_bytes):
+            array_type = c_ubyte * 33
+            proof_pointers[i] = array_type(*node)
+
+        leaf_pointer = (c_ubyte * 32)(*leaf)
+
+        return self.lib.verify(proof_pointers, len_proof, leaf_pointer)

merkly/accelerator/src/lib.rs

@@ -0,0 +1,3 @@
+pub mod merkle_proof;
+pub mod merkle_root;
+pub mod utils;