feat(avm): merkle tree gadget

barretenberg/cpp/pil/avm/gadgets/merkle_tree.pil

@@ -0,0 +1,57 @@
+include "./poseidon2_full.pil";
+
+// Handles membership and insertion into a merkle tree
+namespace merkle_tree(256);
+
+    pol commit sel_merkle_tree;
+    sel_merkle_tree * (1 - sel_merkle_tree) = 0;
+    // Gotta stop using clk for things that are more like foreign keys
+    pol commit clk;
+    // Inputs to the gadget
+    pol commit leaf_value;
+    pol commit leaf_index;
+    pol commit path_len;
+    pol commit expected_tree_root;
+    // These are all hinted
+    pol commit sibling_value;
+
+    // If we are not done, the path_len decrements by 1
+    sel_merkle_tree * (1 - latch) * (path_len' - path_len + 1) = 0;
+
+    pol commit latch;
+    latch * (1 - latch) = 0;
+    pol commit path_len_inv;
+    // latch == 1 when the path_len == 0
+    sel_merkle_tree * (path_len * (latch * (1 - path_len_inv) + path_len_inv) - 1 + latch) = 0;
+
+    // TODO: Constrain that this is indeed even.
+    // Note this might be covered by the halving of the leaf index if the original index is constrained.
+    pol commit leaf_index_is_even;
+    leaf_index_is_even * (1 - leaf_index_is_even) = 0;
+    pol LEAF_INDEX_IS_ODD = (1 - leaf_index_is_even);
+    // If we are not done, the next leaf index is half the current leaf index;
+    // We don't need to worry about underflowing the field since (leaf_index - LEAF_INDEX_IS_ODD)
+    // wil be even (over the integers) and as the field is not of characteristic 2, leaf_index' == leaf_index / 2 over the integers
+    sel_merkle_tree * (1 - latch) * (leaf_index' * 2 + LEAF_INDEX_IS_ODD - leaf_index) = 0;
+
+    // These are what are sent to poseidon2
+    // These arrange the leaf_value and sibling_value in the correct order
+    pol commit left_hash;
+    pol commit right_hash;
+    // I dont think these can be safely combined
+    // if the leaf index is even, the leaf value is the left hash and the sibling value is the right hash
+    // vice-versa
+    sel_merkle_tree * leaf_index_is_even * (left_hash - right_hash) + right_hash - leaf_value = 0;
+    sel_merkle_tree * leaf_index_is_even * (right_hash - left_hash) + left_hash - sibling_value = 0;
+    pol commit output_hash;
+
+    // If we are not done, the output hash is the next value in
+    sel_merkle_tree * (1 - latch) * (leaf_value' - output_hash) = 0;
+
+    pol LAST_COMPUTE = sel_merkle_tree * latch;
+    // LAST_COMPUTE will be used in permutations to other traces
+
+    // Permutation to the full poseidon2 gadget
+    #[PERM_MERKLE_POSEIDON2]
+    sel_merkle_tree { clk, left_hash, right_hash, output_hash } is
+    poseidon2_full.sel_merkle_tree {poseidon2_full.clk, poseidon2_full.input_0, poseidon2_full.input_1, poseidon2_full.output };

barretenberg/cpp/pil/avm/gadgets/poseidon2_full.pil

@@ -87,3 +87,5 @@ namespace poseidon2_full(256);
         { poseidon2.clk, poseidon2.a_0, poseidon2.a_1, poseidon2.a_2, poseidon2.a_3,
           poseidon2.b_0, poseidon2.b_1, poseidon2.b_2, poseidon2.b_3 };
 
+    // ======== Merkle Tree Selector ======================
+    pol commit sel_merkle_tree;

barretenberg/cpp/pil/avm/main.pil

@@ -13,6 +13,7 @@ include "gadgets/poseidon2.pil";
 include "gadgets/poseidon2_full.pil";
 include "gadgets/keccakf1600.pil";
 include "gadgets/mem_slice.pil";
+include "gadgets/merkle_tree.pil";
 
 namespace main(256);
     //===== CONSTANT POLYNOMIALS ==================================================

barretenberg/cpp/src/barretenberg/vm/avm/generated/circuit_builder.cpp

@@ -2,8 +2,6 @@
 #include "barretenberg/vm/avm/generated/circuit_builder.hpp"
 
 #include <mutex>
-#include <set>
-#include <unordered_map>
 
 #include "barretenberg/common/constexpr_utils.hpp"
 #include "barretenberg/common/thread.hpp"
@@ -24,23 +22,6 @@ AvmCircuitBuilder::ProverPolynomials AvmCircuitBuilder::compute_polynomials() co
     ASSERT(num_rows <= circuit_subgroup_size);
     ProverPolynomials polys;
 
-    // We create a mapping between the polynomial index and the corresponding column index when row
-    // is expressed as a vector, i.e., column of the trace matrix.
-    std::unordered_map<std::string, size_t> names_to_col_idx;
-    const auto names = Row::names();
-    for (size_t i = 0; i < names.size(); i++) {
-        names_to_col_idx[names[i]] = i;
-    }
-
-    const auto labels = polys.get_unshifted_labels();
-    const size_t num_unshifted = labels.size();
-
-    // Mapping
-    std::vector<size_t> polys_to_cols_unshifted_idx(num_unshifted);
-    for (size_t i = 0; i < num_unshifted; i++) {
-        polys_to_cols_unshifted_idx[i] = names_to_col_idx.at(labels[i]);
-    }
-
     // Allocate mem for each column
     AVM_TRACK_TIME("circuit_builder/init_polys_to_be_shifted", ({
                        for (auto& poly : polys.get_to_be_shifted()) {
@@ -49,48 +30,15 @@ AvmCircuitBuilder::ProverPolynomials AvmCircuitBuilder::compute_polynomials() co
                                               /*make shiftable with offset*/ 1 };
                        }
                    }));
-
     // catch-all with fully formed polynomials
     AVM_TRACK_TIME(
         "circuit_builder/init_polys_unshifted", ({
             auto unshifted = polys.get_unshifted();
-
-            // An array which stores for each column of the trace the smallest size of the
-            // truncated column containing all non-zero elements.
-            // It is used to allocate the polynomials without memory overhead for the tail of zeros.
-            std::array<size_t, Row::SIZE> col_nonzero_size{};
-
-            // Computation of size of columns.
-            // Non-parallel version takes 0.5 second for a trace size of 200k rows.
-            // A parallel version might be considered in the future.
-            for (size_t i = 0; i < num_rows; i++) {
-                const auto row = rows[i].as_vector();
-                for (size_t col = 0; col < Row::SIZE; col++) {
-                    if (!row[col].is_zero()) {
-                        col_nonzero_size[col] = i + 1;
-                    }
-                }
-            }
-
-            // Set of the labels for derived/inverse polynomials.
-            const auto derived_labels = polys.get_derived_labels();
-            std::set<std::string> derived_labels_set(derived_labels.begin(), derived_labels.end());
-
-            bb::parallel_for(num_unshifted, [&](size_t i) {
+            bb::parallel_for(unshifted.size(), [&](size_t i) {
                 auto& poly = unshifted[i];
-                const auto col_idx = polys_to_cols_unshifted_idx[i];
-                size_t col_size = 0;
-
-                // We fully allocate the inverse polynomials. We leave this potential memory optimization for later.
-                if (derived_labels_set.contains(labels[i])) {
-                    col_size = num_rows;
-                } else {
-                    col_size = col_nonzero_size[col_idx];
-                }
-
                 if (poly.is_empty()) {
                     // Not set above
-                    poly = Polynomial{ /*memory size*/ col_size, /*largest possible index*/ circuit_subgroup_size };
+                    poly = Polynomial{ /*memory size*/ num_rows, /*largest possible index*/ circuit_subgroup_size };
                 }
             });
         }));
@@ -423,6 +371,19 @@ AvmCircuitBuilder::ProverPolynomials AvmCircuitBuilder::compute_polynomials() co
                 polys.mem_tsp.set_if_valid_index(i, rows[i].mem_tsp);
                 polys.mem_val.set_if_valid_index(i, rows[i].mem_val);
                 polys.mem_w_in_tag.set_if_valid_index(i, rows[i].mem_w_in_tag);
+                polys.merkle_tree_clk.set_if_valid_index(i, rows[i].merkle_tree_clk);
+                polys.merkle_tree_expected_tree_root.set_if_valid_index(i, rows[i].merkle_tree_expected_tree_root);
+                polys.merkle_tree_latch.set_if_valid_index(i, rows[i].merkle_tree_latch);
+                polys.merkle_tree_leaf_index.set_if_valid_index(i, rows[i].merkle_tree_leaf_index);
+                polys.merkle_tree_leaf_index_is_even.set_if_valid_index(i, rows[i].merkle_tree_leaf_index_is_even);
+                polys.merkle_tree_leaf_value.set_if_valid_index(i, rows[i].merkle_tree_leaf_value);
+                polys.merkle_tree_left_hash.set_if_valid_index(i, rows[i].merkle_tree_left_hash);
+                polys.merkle_tree_output_hash.set_if_valid_index(i, rows[i].merkle_tree_output_hash);
+                polys.merkle_tree_path_len.set_if_valid_index(i, rows[i].merkle_tree_path_len);
+                polys.merkle_tree_path_len_inv.set_if_valid_index(i, rows[i].merkle_tree_path_len_inv);
+                polys.merkle_tree_right_hash.set_if_valid_index(i, rows[i].merkle_tree_right_hash);
+                polys.merkle_tree_sel_merkle_tree.set_if_valid_index(i, rows[i].merkle_tree_sel_merkle_tree);
+                polys.merkle_tree_sibling_value.set_if_valid_index(i, rows[i].merkle_tree_sibling_value);
                 polys.poseidon2_B_10_0.set_if_valid_index(i, rows[i].poseidon2_B_10_0);
                 polys.poseidon2_B_10_1.set_if_valid_index(i, rows[i].poseidon2_B_10_1);
                 polys.poseidon2_B_10_2.set_if_valid_index(i, rows[i].poseidon2_B_10_2);
@@ -714,6 +675,7 @@ AvmCircuitBuilder::ProverPolynomials AvmCircuitBuilder::compute_polynomials() co
                     i, rows[i].poseidon2_full_num_perm_rounds_rem_inv);
                 polys.poseidon2_full_output.set_if_valid_index(i, rows[i].poseidon2_full_output);
                 polys.poseidon2_full_padding.set_if_valid_index(i, rows[i].poseidon2_full_padding);
+                polys.poseidon2_full_sel_merkle_tree.set_if_valid_index(i, rows[i].poseidon2_full_sel_merkle_tree);
                 polys.poseidon2_full_sel_poseidon.set_if_valid_index(i, rows[i].poseidon2_full_sel_poseidon);
                 polys.poseidon2_full_start_poseidon.set_if_valid_index(i, rows[i].poseidon2_full_start_poseidon);
                 polys.poseidon2_input_addr.set_if_valid_index(i, rows[i].poseidon2_input_addr);