Skip to content

Commit a1036c0

Browse files
committed
Implement get-compact-range using RFC 6962 methods
This is a proof of concept change demonstrating that it is possible to obtain arbitrary compact ranges from a Merkle tree log that restricts itself only to endpoints represented in RFC 6962, in constant time interaction complexity. Specifically, it is possible to obtain comact range [begin, end) by calling "get consistency proof" endpoints <= 2 times for carefully crafted tree sizes. In a few cases where it is impossible to get certain hashes, this approach falls back to calling the "get entries" endpoint 1 time to obtain between 1-3 entries and reconstruct the compact range. Overall, the interaction with the log is limited by 2 calls, and each call is limited in size.
1 parent c120179 commit a1036c0

File tree

5 files changed

+299
-0
lines changed

5 files changed

+299
-0
lines changed

exp/README.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
Experimental
2+
------------
3+
4+
This directory contains a Go module with experimental features not included into
5+
the main Go module of this repository. These must be used with caution.
6+
7+
The idea of this module is similar to Go's https://pkg.go.dev/golang.org/x/exp.

exp/get_compact_range.go

Lines changed: 153 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,153 @@
1+
// Copyright 2022 Google LLC. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package merkle
16+
17+
import (
18+
"fmt"
19+
20+
"github.com/transparency-dev/merkle/compact"
21+
"github.com/transparency-dev/merkle/proof"
22+
)
23+
24+
type HashGetter interface {
25+
GetConsistencyProof(first, second uint64) ([][]byte, error)
26+
GetLeafHashes(begin, end uint64) ([][]byte, error)
27+
}
28+
29+
func GetCompactRange(rf *compact.RangeFactory, begin, end, size uint64, hg HashGetter) (*compact.Range, error) {
30+
if begin > size || end > size {
31+
return nil, fmt.Errorf("[%d, %d) out of range in %d", begin, end, size)
32+
}
33+
if begin >= end {
34+
return rf.NewEmptyRange(begin), nil
35+
}
36+
37+
if size <= 3 || end == 1 {
38+
hashes, err := hg.GetLeafHashes(begin, end)
39+
if err != nil {
40+
return nil, fmt.Errorf("GetLeafHashes(%d, %d): %v", begin, end, err)
41+
}
42+
if got, want := uint64(len(hashes)), end-begin; got != want {
43+
return nil, fmt.Errorf("GetLeafHashes(%d, %d): %d hashes, want %d", begin, end, got, want)
44+
}
45+
r := rf.NewEmptyRange(begin)
46+
for _, h := range hashes {
47+
if err := r.Append(h, nil); err != nil {
48+
return nil, fmt.Errorf("Append: %v", err)
49+
}
50+
}
51+
return r, nil
52+
}
53+
// size >= 4 && end >= 2
54+
55+
known := make(map[compact.NodeID][]byte)
56+
57+
store := func(nodes proof.Nodes, hashes [][]byte) error {
58+
_, b, e := nodes.Ephem()
59+
wantSize := len(nodes.IDs) - (e - b)
60+
if b != e {
61+
wantSize++
62+
}
63+
if got := len(hashes); got != wantSize {
64+
return fmt.Errorf("proof size mismatch: got %d, want %d", got, wantSize)
65+
}
66+
67+
idx := 0
68+
for _, hash := range hashes {
69+
if idx == b && b+1 < e {
70+
idx = e - 1
71+
continue
72+
}
73+
known[nodes.IDs[idx]] = hash
74+
idx++
75+
}
76+
return nil
77+
}
78+
79+
newRange := func(begin, end uint64) (*compact.Range, error) {
80+
size := compact.RangeSize(begin, end)
81+
ids := compact.RangeNodes(begin, end, make([]compact.NodeID, 0, size))
82+
hashes := make([][]byte, 0, len(ids))
83+
for _, id := range ids {
84+
if hash, ok := known[id]; ok {
85+
hashes = append(hashes, hash)
86+
} else {
87+
return nil, fmt.Errorf("hash not known: %+v", id)
88+
}
89+
}
90+
return rf.NewRange(begin, end, hashes)
91+
}
92+
93+
fetch := func(first, second uint64) error {
94+
nodes, err := proof.Consistency(first, second)
95+
if err != nil {
96+
return fmt.Errorf("proof.Consistency: %v", err)
97+
}
98+
hashes, err := hg.GetConsistencyProof(first, second)
99+
if err != nil {
100+
return fmt.Errorf("GetConsistencyProof(%d, %d): %v", first, second, err)
101+
}
102+
store(nodes, hashes)
103+
return nil
104+
}
105+
106+
mid, _ := compact.Decompose(begin, end)
107+
mid += begin
108+
if err := fetch(begin, mid); err != nil {
109+
return nil, err
110+
}
111+
112+
if begin == 0 && end == 2 || end == 3 {
113+
if err := fetch(3, 4); err != nil {
114+
return nil, err
115+
}
116+
}
117+
if end <= 3 {
118+
return newRange(begin, end)
119+
}
120+
// end >= 4
121+
122+
if (end-1)&(end-2) != 0 { // end-1 is not a power of 2.
123+
if err := fetch(end-1, end); err != nil {
124+
return nil, err
125+
}
126+
r, err := newRange(begin, end-1)
127+
if err != nil {
128+
return nil, err
129+
}
130+
if err := r.Append(known[compact.NewNodeID(0, end-1)], nil); err != nil {
131+
return nil, fmt.Errorf("Append: %v", err)
132+
}
133+
return r, nil
134+
}
135+
136+
// At this point: end >= 4, end-1 is a power of 2; thus, end-2 is not a power of 2.
137+
if err := fetch(end-2, end); err != nil {
138+
return nil, err
139+
}
140+
r := rf.NewEmptyRange(begin)
141+
if end-2 > begin {
142+
var err error
143+
if r, err = newRange(begin, end-2); err != nil {
144+
return nil, err
145+
}
146+
}
147+
for index := r.End(); index < end; index++ {
148+
if err := r.Append(known[compact.NewNodeID(0, index)], nil); err != nil {
149+
return nil, fmt.Errorf("Append: %v", err)
150+
}
151+
}
152+
return r, nil
153+
}

exp/get_compact_range_test.go

Lines changed: 130 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
// Copyright 2022 Google LLC. All Rights Reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
package merkle_test
16+
17+
import (
18+
"fmt"
19+
"testing"
20+
21+
"github.com/google/go-cmp/cmp"
22+
"github.com/transparency-dev/merkle"
23+
"github.com/transparency-dev/merkle/compact"
24+
"github.com/transparency-dev/merkle/proof"
25+
)
26+
27+
func TestGetCompactRange(t *testing.T) {
28+
rf := compact.RangeFactory{Hash: func(left, right []byte) []byte {
29+
return append(append(make([]byte, 0, len(left)+len(right)), left...), right...)
30+
}}
31+
tr := newTree(t, 256, &rf)
32+
33+
test := func(begin, end, size uint64) {
34+
t.Run(fmt.Sprintf("%d:%d_%d", size, begin, end), func(t *testing.T) {
35+
got, err := merkle.GetCompactRange(&rf, begin, end, size, tr)
36+
if err != nil {
37+
t.Fatalf("GetCompactRange: %v", err)
38+
}
39+
want, err := tr.getCompactRange(begin, end)
40+
if err != nil {
41+
t.Fatalf("GetCompactRange: %v", err)
42+
}
43+
if diff := cmp.Diff(got, want); diff != "" {
44+
t.Fatalf("Diff: %s", diff)
45+
}
46+
})
47+
}
48+
49+
for begin := uint64(0); begin <= tr.size; begin++ {
50+
for end := begin; end <= tr.size; end++ {
51+
for size := end; size < end+5 && size < tr.size; size++ {
52+
test(begin, end, size)
53+
}
54+
test(begin, end, tr.size)
55+
}
56+
}
57+
}
58+
59+
type tree struct {
60+
rf *compact.RangeFactory
61+
size uint64
62+
nodes map[compact.NodeID][]byte
63+
}
64+
65+
func newTree(t *testing.T, size uint64, rf *compact.RangeFactory) *tree {
66+
hash := func(leaf uint64) []byte {
67+
if leaf >= 256 {
68+
t.Fatalf("leaf %d not supported in this test", leaf)
69+
}
70+
return []byte{byte(leaf)}
71+
}
72+
73+
nodes := make(map[compact.NodeID][]byte, size*2-1)
74+
r := rf.NewEmptyRange(0)
75+
for i := uint64(0); i < size; i++ {
76+
nodes[compact.NewNodeID(0, i)] = hash(i)
77+
if err := r.Append(hash(i), func(id compact.NodeID, hash []byte) {
78+
nodes[id] = hash
79+
}); err != nil {
80+
t.Fatalf("Append: %v", err)
81+
}
82+
}
83+
return &tree{rf: rf, size: size, nodes: nodes}
84+
}
85+
86+
func (t *tree) GetConsistencyProof(first, second uint64) ([][]byte, error) {
87+
if first > t.size || second > t.size {
88+
return nil, fmt.Errorf("%d or %d is beyond %d", first, second, t.size)
89+
}
90+
nodes, err := proof.Consistency(first, second)
91+
if err != nil {
92+
return nil, err
93+
}
94+
hashes, err := t.getNodes(nodes.IDs)
95+
if err != nil {
96+
return nil, err
97+
}
98+
return nodes.Rehash(hashes, t.rf.Hash)
99+
}
100+
101+
func (t *tree) GetLeafHashes(begin, end uint64) ([][]byte, error) {
102+
if begin >= end {
103+
return nil, nil
104+
}
105+
ids := make([]compact.NodeID, 0, end-begin)
106+
for i := begin; i < end; i++ {
107+
ids = append(ids, compact.NewNodeID(0, i))
108+
}
109+
return t.getNodes(ids)
110+
}
111+
112+
func (t *tree) getCompactRange(begin, end uint64) (*compact.Range, error) {
113+
hashes, err := t.getNodes(compact.RangeNodes(begin, end))
114+
if err != nil {
115+
return nil, err
116+
}
117+
return t.rf.NewRange(begin, end, hashes)
118+
}
119+
120+
func (t *tree) getNodes(ids []compact.NodeID) ([][]byte, error) {
121+
hashes := make([][]byte, len(ids))
122+
for i, id := range ids {
123+
if hash, ok := t.nodes[id]; ok {
124+
hashes[i] = hash
125+
} else {
126+
return nil, fmt.Errorf("node %+v not found", id)
127+
}
128+
}
129+
return hashes, nil
130+
}

exp/go.mod

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/transparency-dev/merkle/exp
2+
3+
go 1.16
4+
5+
require github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad // indirect

exp/go.sum

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
2+
github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad h1:82yvTO+VijfWulMsMQvqQSZ0zNEAgmEUeBG+ArrO9Js=
3+
github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad/go.mod h1:B8FIw5LTq6DaULoHsVFRzYIUDkl8yuSwCdZnOZGKL/A=
4+
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

0 commit comments

Comments
 (0)