Skip to content

Commit

Permalink
Add Maybe to the end bound of proofs (Part 1) (#1793)
Browse files Browse the repository at this point in the history 
Co-authored-by: Dan Laine <daniel.laine@avalabs.org>
  • Loading branch information
@dboehm-avalabs
dboehm-avalabs and Dan Laine committed Aug 7, 2023
1 parent 1bb3a00 commit 8579d7f
Show file tree
Hide file tree
Showing 21 changed files with 496 additions and 428 deletions.
422 changes: 213 additions & 209 deletions proto/pb/sync/sync.pb.go
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions proto/sync/sync.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ message SyncGetChangeProofRequest {
bytes start_root_hash = 1;
bytes end_root_hash = 2;
bytes start_key = 3;
bytes end_key = 4;
MaybeBytes end_key = 4;
uint32 key_limit = 5;
uint32 bytes_limit = 6;
}
Expand Down Expand Up @@ -96,7 +96,7 @@ message CommitChangeProofRequest {
message SyncGetRangeProofRequest {
bytes root_hash = 1;
bytes start_key = 2;
bytes end_key = 3;
MaybeBytes end_key = 3;
uint32 key_limit = 4;
uint32 bytes_limit = 5;
}
Expand Down
39 changes: 19 additions & 20 deletions x/merkledb/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ type ChangeProofer interface {
// - [proof] is non-empty iff [proof.HadRootsInHistory].
// - All keys in [proof.KeyValues] and [proof.DeletedKeys] are in [start, end].
// If [start] is empty, all keys are considered > [start].
// If [end] is empty, all keys are considered < [end].
// If [end] is nothing, all keys are considered < [end].
// - [proof.KeyValues] and [proof.DeletedKeys] are sorted in order of increasing key.
// - [proof.StartProof] and [proof.EndProof] are well-formed.
// - When the keys in [proof.KeyValues] are added to [db] and the keys in [proof.DeletedKeys]
Expand All @@ -83,7 +83,7 @@ type ChangeProofer interface {
ctx context.Context,
proof *ChangeProof,
start []byte,
end []byte,
end Maybe[[]byte],
expectedEndRootID ids.ID,
) error

Expand Down Expand Up @@ -582,20 +582,22 @@ func (db *merkleDB) GetChangeProof(
})
}

largestKey := end
largestKey := Nothing[[]byte]()
if len(result.KeyChanges) > 0 {
largestKey = result.KeyChanges[len(result.KeyChanges)-1].Key
largestKey = Some(result.KeyChanges[len(result.KeyChanges)-1].Key)
} else if len(end) > 0 {
largestKey = Some(end)
}

// Since we hold [db.commitlock] we must still have sufficient
// history to recreate the trie at [endRootID].
historicalView, err := db.getHistoricalViewForRange(endRootID, start, largestKey)
historicalView, err := db.getHistoricalViewForRange(endRootID, start, largestKey.Value())
if err != nil {
return nil, err
}

if len(largestKey) > 0 {
endProof, err := historicalView.getProof(ctx, largestKey)
if largestKey.HasValue() {
endProof, err := historicalView.getProof(ctx, largestKey.Value())
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -958,10 +960,10 @@ func (db *merkleDB) VerifyChangeProof(
ctx context.Context,
proof *ChangeProof,
start []byte,
end []byte,
end Maybe[[]byte],
expectedEndRootID ids.ID,
) error {
if len(end) > 0 && bytes.Compare(start, end) > 0 {
if end.HasValue() && bytes.Compare(start, end.Value()) > 0 {
return ErrStartAfterEnd
}

Expand All @@ -978,7 +980,7 @@ func (db *merkleDB) VerifyChangeProof(
switch {
case proof.Empty():
return ErrNoMerkleProof
case len(end) > 0 && len(proof.EndProof) == 0:
case end.HasValue() && len(proof.EndProof) == 0:
// We requested an end proof but didn't get one.
return ErrNoEndProof
case len(start) > 0 && len(proof.StartProof) == 0 && len(proof.EndProof) == 0:
Expand All @@ -1004,17 +1006,18 @@ func (db *merkleDB) VerifyChangeProof(
// Find the greatest key in [proof.KeyChanges]
// Note that [proof.EndProof] is a proof for this key.
// [largestPath] is also used when we add children of proof nodes to [trie] below.
largestKey := end
largestPath := Nothing[path]()
if len(proof.KeyChanges) > 0 {
// If [proof] has key-value pairs, we should insert children
// greater than [end] to ancestors of the node containing [end]
// so that we get the expected root ID.
largestKey = proof.KeyChanges[len(proof.KeyChanges)-1].Key
largestPath = Some(newPath(proof.KeyChanges[len(proof.KeyChanges)-1].Key))
} else if end.HasValue() {
largestPath = Some(newPath(end.Value()))
}
largestPath := newPath(largestKey)

// Make sure the end proof, if given, is well-formed.
if err := verifyProofPath(proof.EndProof, largestPath); err != nil {
if err := verifyProofPath(proof.EndProof, largestPath.Value()); err != nil {
return err
}

Expand Down Expand Up @@ -1074,23 +1077,19 @@ func (db *merkleDB) VerifyChangeProof(
if len(smallestPath) > 0 {
insertChildrenLessThan = Some(smallestPath)
}
insertChildrenGreaterThan := Nothing[path]()
if len(largestPath) > 0 {
insertChildrenGreaterThan = Some(largestPath)
}
if err := addPathInfo(
view,
proof.StartProof,
insertChildrenLessThan,
insertChildrenGreaterThan,
largestPath,
); err != nil {
return err
}
if err := addPathInfo(
view,
proof.EndProof,
insertChildrenLessThan,
insertChildrenGreaterThan,
largestPath,
); err != nil {
return err
}
Expand Down
12 changes: 10 additions & 2 deletions x/merkledb/db_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -770,10 +770,14 @@ func runRandDBTest(require *require.Assertions, r *rand.Rand, rt randTest) {
}
rangeProof, err := db.GetRangeProofAtRoot(context.Background(), root, step.key, step.value, 100)
require.NoError(err)
end := Nothing[[]byte]()
if len(step.value) > 0 {
end = Some(step.value)
}
require.NoError(rangeProof.Verify(
context.Background(),
step.key,
step.value,
end,
root,
))
require.LessOrEqual(len(rangeProof.KeyValues), 100)
Expand All @@ -791,11 +795,15 @@ func runRandDBTest(require *require.Assertions, r *rand.Rand, rt randTest) {
require.NoError(err)
changeProofDB, err := getBasicDB()
require.NoError(err)
end := Nothing[[]byte]()
if len(step.value) > 0 {
end = Some(step.value)
}
require.NoError(changeProofDB.VerifyChangeProof(
context.Background(),
changeProof,
step.key,
step.value,
end,
root,
))
require.LessOrEqual(len(changeProof.KeyChanges), 100)
Expand Down
38 changes: 19 additions & 19 deletions x/merkledb/history_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,15 +36,15 @@ func Test_History_Simple(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("key"), []byte("value0")))
require.NoError(batch.Write())
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("key1"), []byte("value1")))
Expand All @@ -53,15 +53,15 @@ func Test_History_Simple(t *testing.T) {
newProof, err = db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("k"), []byte("v")))
require.NoError(batch.Write())
newProof, err = db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Delete([]byte("k")))
Expand All @@ -77,7 +77,7 @@ func Test_History_Simple(t *testing.T) {
newProof, err = db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_Large(t *testing.T) {
Expand Down Expand Up @@ -137,7 +137,7 @@ func Test_History_Large(t *testing.T) {
require.NoError(err)
require.NotNil(proof)

require.NoError(proof.Verify(context.Background(), nil, nil, roots[0]))
require.NoError(proof.Verify(context.Background(), nil, Nothing[[]byte](), roots[0]))
}
}

Expand Down Expand Up @@ -233,7 +233,7 @@ func Test_History_Trigger_History_Queue_Looping(t *testing.T) {
require.NoError(origProof.Verify(
context.Background(),
[]byte("k"),
[]byte("key3"),
Some([]byte("key3")),
origRootID,
))

Expand All @@ -249,7 +249,7 @@ func Test_History_Trigger_History_Queue_Looping(t *testing.T) {
require.NoError(newProof.Verify(
context.Background(),
[]byte("k"),
[]byte("key3"),
Some([]byte("key3")),
origRootID,
))

Expand Down Expand Up @@ -332,7 +332,7 @@ func Test_History_RepeatedRoot(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("key1"), []byte("other")))
Expand All @@ -342,7 +342,7 @@ func Test_History_RepeatedRoot(t *testing.T) {
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

// revert state to be the same as in orig proof
batch = db.NewBatch()
Expand All @@ -354,7 +354,7 @@ func Test_History_RepeatedRoot(t *testing.T) {
newProof, err = db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_ExcessDeletes(t *testing.T) {
Expand All @@ -374,7 +374,7 @@ func Test_History_ExcessDeletes(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Delete([]byte("key1")))
Expand All @@ -386,7 +386,7 @@ func Test_History_ExcessDeletes(t *testing.T) {
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_DontIncludeAllNodes(t *testing.T) {
Expand All @@ -406,15 +406,15 @@ func Test_History_DontIncludeAllNodes(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("z"), []byte("z")))
require.NoError(batch.Write())
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_Branching2Nodes(t *testing.T) {
Expand All @@ -434,15 +434,15 @@ func Test_History_Branching2Nodes(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("k"), []byte("v")))
require.NoError(batch.Write())
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_Branching3Nodes(t *testing.T) {
Expand All @@ -462,15 +462,15 @@ func Test_History_Branching3Nodes(t *testing.T) {
require.NoError(err)
require.NotNil(origProof)
origRootID := db.root.id
require.NoError(origProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(origProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))

batch = db.NewBatch()
require.NoError(batch.Put([]byte("key321"), []byte("value321")))
require.NoError(batch.Write())
newProof, err := db.GetRangeProofAtRoot(context.Background(), origRootID, []byte("k"), []byte("key3"), 10)
require.NoError(err)
require.NotNil(newProof)
require.NoError(newProof.Verify(context.Background(), []byte("k"), []byte("key3"), origRootID))
require.NoError(newProof.Verify(context.Background(), []byte("k"), Some([]byte("key3")), origRootID))
}

func Test_History_MaxLength(t *testing.T) {
Expand Down
29 changes: 27 additions & 2 deletions x/merkledb/maybe.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@

package merkledb

import "golang.org/x/exp/slices"
import (
"bytes"

"golang.org/x/exp/slices"
)

// Maybe T = Some T | Nothing.
// A data wrapper that allows values to be something [Some T] or nothing [Nothing].
Expand All @@ -29,11 +33,16 @@ func Nothing[T any]() Maybe[T] {
return Maybe[T]{}
}

// Returns true iff [m] has a value.
// Returns false iff [m] has a value.
func (m Maybe[T]) IsNothing() bool {
return !m.hasValue
}

// Returns true iff [m] has a value.
func (m Maybe[T]) HasValue() bool {
return m.hasValue
}

// Returns the value of [m].
func (m Maybe[T]) Value() T {
return m.value
Expand All @@ -45,3 +54,19 @@ func Clone(m Maybe[[]byte]) Maybe[[]byte] {
}
return Some(slices.Clone(m.value))
}

// MaybeBytesEquals returns true iff [a] and [b] are equal.
func MaybeBytesEquals(a, b Maybe[[]byte]) bool {
aNothing := a.IsNothing()
bNothing := b.IsNothing()

if aNothing {
return bNothing
}

if bNothing {
return false
}

return bytes.Equal(a.Value(), b.Value())
}
2 changes: 1 addition & 1 deletion x/merkledb/mock_db.go
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 8579d7f

Please sign in to comment.