tornadocontrib/bsc
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

trie: extend range proof (#21250)

Browse Source 
* trie: support non-existent right proof

* trie: improve test

* trie: minor linter fix

Co-authored-by: Péter Szilágyi <peterke@gmail.com>
This commit is contained in:
gary rong 2020-09-23 17:44:09 +08:00 committed by GitHub
parent 0921f8a74f
commit e5defccd58
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 397 additions and 184 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

197
trie/proof.go
View File

@ -129,10 +129,11 @@ func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader)
} }
} }
// proofToPath converts a merkle proof to trie node path. // proofToPath converts a merkle proof to trie node path. The main purpose of
// The main purpose of this function is recovering a node // this function is recovering a node path from the merkle proof stream. All
// path from the merkle proof stream. All necessary nodes // necessary nodes will be resolved and leave the remaining as hashnode.
// will be resolved and leave the remaining as hashnode. //
// The given edge proof is allowed to be an existent or non-existent proof.
func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyValueReader, allowNonExistent bool) (node, []byte, error) { func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyValueReader, allowNonExistent bool) (node, []byte, error) {
// resolveNode retrieves and resolves trie node from merkle proof stream // resolveNode retrieves and resolves trie node from merkle proof stream
resolveNode := func(hash common.Hash) (node, error) { resolveNode := func(hash common.Hash) (node, error) {
@ -205,54 +206,61 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV
} }
// unsetInternal removes all internal node references(hashnode, embedded node). // unsetInternal removes all internal node references(hashnode, embedded node).
// It should be called after a trie is constructed with two edge proofs. Also // It should be called after a trie is constructed with two edge paths. Also
// the given boundary keys must be the one used to construct the edge proofs. // the given boundary keys must be the one used to construct the edge paths.
// //
// It's the key step for range proof. All visited nodes should be marked dirty // It's the key step for range proof. All visited nodes should be marked dirty
// since the node content might be modified. Besides it can happen that some // since the node content might be modified. Besides it can happen that some
// fullnodes only have one child which is disallowed. But if the proof is valid, // fullnodes only have one child which is disallowed. But if the proof is valid,
// the missing children will be filled, otherwise it will be thrown anyway. // the missing children will be filled, otherwise it will be thrown anyway.
//
// Note we have the assumption here the given boundary keys are different
// and right is larger than left.
func unsetInternal(n node, left []byte, right []byte) error { func unsetInternal(n node, left []byte, right []byte) error {
left, right = keybytesToHex(left), keybytesToHex(right) left, right = keybytesToHex(left), keybytesToHex(right)
// todo(rjl493456442) different length edge keys should be supported
if len(left) != len(right) {
return errors.New("inconsistent edge path")
}
// Step down to the fork point. There are two scenarios can happen: // Step down to the fork point. There are two scenarios can happen:
// - the fork point is a shortnode: the left proof MUST point to a // - the fork point is a shortnode: either the key of left proof or
// non-existent key and the key doesn't match with the shortnode // right proof doesn't match with shortnode's key.
// - the fork point is a fullnode: the left proof can point to an // - the fork point is a fullnode: both two edge proofs are allowed
// existent key or not. // to point to a non-existent key.
var ( var (
pos = 0 pos = 0
parent node parent node
// fork indicator, 0 means no fork, -1 means proof is less, 1 means proof is greater
shortForkLeft, shortForkRight int
) )
findFork: findFork:
for { for {
switch rn := (n).(type) { switch rn := (n).(type) {
case *shortNode: case *shortNode:
// The right proof must point to an existent key.
if len(right)-pos < len(rn.Key) || !bytes.Equal(rn.Key, right[pos:pos+len(rn.Key)]) {
return errors.New("invalid edge path")
}
rn.flags = nodeFlag{dirty: true} rn.flags = nodeFlag{dirty: true}
// Special case, the non-existent proof points to the same path
// as the existent proof, but the path of existent proof is longer. // If either the key of left proof or right proof doesn't match with
// In this case, the fork point is this shortnode. // shortnode, stop here and the forkpoint is the shortnode.
if len(left)-pos < len(rn.Key) || !bytes.Equal(rn.Key, left[pos:pos+len(rn.Key)]) { if len(left)-pos < len(rn.Key) {
shortForkLeft = bytes.Compare(left[pos:], rn.Key)
} else {
shortForkLeft = bytes.Compare(left[pos:pos+len(rn.Key)], rn.Key)
}
if len(right)-pos < len(rn.Key) {
shortForkRight = bytes.Compare(right[pos:], rn.Key)
} else {
shortForkRight = bytes.Compare(right[pos:pos+len(rn.Key)], rn.Key)
}
if shortForkLeft != 0 || shortForkRight != 0 {
break findFork break findFork
} }
parent = n parent = n
n, pos = rn.Val, pos+len(rn.Key) n, pos = rn.Val, pos+len(rn.Key)
case *fullNode: case *fullNode:
leftnode, rightnode := rn.Children[left[pos]], rn.Children[right[pos]]
// The right proof must point to an existent key.
if rightnode == nil {
return errors.New("invalid edge path")
}
rn.flags = nodeFlag{dirty: true} rn.flags = nodeFlag{dirty: true}
if leftnode != rightnode {
// If either the node pointed by left proof or right proof is nil,
// stop here and the forkpoint is the fullnode.
leftnode, rightnode := rn.Children[left[pos]], rn.Children[right[pos]]
if leftnode == nil || rightnode == nil || leftnode != rightnode {
break findFork break findFork
} }
parent = n parent = n
@ -263,12 +271,42 @@ findFork:
} }
switch rn := n.(type) { switch rn := n.(type) {
case *shortNode: case *shortNode:
// There can have these five scenarios:
// - both proofs are less than the trie path => no valid range
// - both proofs are greater than the trie path => no valid range
// - left proof is less and right proof is greater => valid range, unset the shortnode entirely
// - left proof points to the shortnode, but right proof is greater
// - right proof points to the shortnode, but left proof is less
if shortForkLeft == -1 && shortForkRight == -1 {
return errors.New("empty range")
}
if shortForkLeft == 1 && shortForkRight == 1 {
return errors.New("empty range")
}
if shortForkLeft != 0 && shortForkRight != 0 {
parent.(*fullNode).Children[left[pos-1]] = nil
return nil
}
// Only one proof points to non-existent key.
if shortForkRight != 0 {
// Unset left proof's path
if _, ok := rn.Val.(valueNode); ok {
parent.(*fullNode).Children[left[pos-1]] = nil
return nil
}
return unset(rn, rn.Val, left[pos:], len(rn.Key), false)
}
if shortForkLeft != 0 {
// Unset right proof's path.
if _, ok := rn.Val.(valueNode); ok { if _, ok := rn.Val.(valueNode); ok {
parent.(*fullNode).Children[right[pos-1]] = nil parent.(*fullNode).Children[right[pos-1]] = nil
return nil return nil
} }
return unset(rn, rn.Val, right[pos:], len(rn.Key), true) return unset(rn, rn.Val, right[pos:], len(rn.Key), true)
}
return nil
case *fullNode: case *fullNode:
// unset all internal nodes in the forkpoint
for i := left[pos] + 1; i < right[pos]; i++ { for i := left[pos] + 1; i < right[pos]; i++ {
rn.Children[i] = nil rn.Children[i] = nil
} }
@ -285,19 +323,17 @@ findFork:
} }
// unset removes all internal node references either the left most or right most. // unset removes all internal node references either the left most or right most.
// If we try to unset all right most references, it can meet these scenarios: // It can meet these scenarios:
// //
// - The given path is existent in the trie, unset the associated shortnode // - The given path is existent in the trie, unset the associated nodes with the
// specific direction
// - The given path is non-existent in the trie // - The given path is non-existent in the trie
// - the fork point is a fullnode, the corresponding child pointed by path // - the fork point is a fullnode, the corresponding child pointed by path
// is nil, return // is nil, return
// - the fork point is a shortnode, the key of shortnode is less than path, // - the fork point is a shortnode, the shortnode is included in the range,
// keep the entire branch and return. // keep the entire branch and return.
// - the fork point is a shortnode, the key of shortnode is greater than path, // - the fork point is a shortnode, the shortnode is excluded in the range,
// unset the entire branch. // unset the entire branch.
//
// If we try to unset all left most references, then the given path should
// be existent.
func unset(parent node, child node, key []byte, pos int, removeLeft bool) error { func unset(parent node, child node, key []byte, pos int, removeLeft bool) error {
switch cld := child.(type) { switch cld := child.(type) {
case *fullNode: case *fullNode:
@ -317,8 +353,18 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error
if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) {
// Find the fork point, it's an non-existent branch. // Find the fork point, it's an non-existent branch.
if removeLeft { if removeLeft {
return errors.New("invalid right edge proof") if bytes.Compare(cld.Key, key[pos:]) < 0 {
// The key of fork shortnode is less than the path
// (it belongs to the range), unset the entrie
// branch. The parent must be a fullnode.
fn := parent.(*fullNode)
fn.Children[key[pos-1]] = nil
} else {
// The key of fork shortnode is greater than the
// path(it doesn't belong to the range), keep
// it with the cached hash available.
} }
} else {
if bytes.Compare(cld.Key, key[pos:]) > 0 { if bytes.Compare(cld.Key, key[pos:]) > 0 {
// The key of fork shortnode is greater than the // The key of fork shortnode is greater than the
// path(it belongs to the range), unset the entrie // path(it belongs to the range), unset the entrie
@ -330,6 +376,7 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error
// path(it doesn't belong to the range), keep // path(it doesn't belong to the range), keep
// it with the cached hash available. // it with the cached hash available.
} }
}
return nil return nil
} }
if _, ok := cld.Val.(valueNode); ok { if _, ok := cld.Val.(valueNode); ok {
@ -340,11 +387,8 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error
cld.flags = nodeFlag{dirty: true} cld.flags = nodeFlag{dirty: true}
return unset(cld, cld.Val, key, pos+len(cld.Key), removeLeft) return unset(cld, cld.Val, key, pos+len(cld.Key), removeLeft)
case nil: case nil:
// If the node is nil, it's a child of the fork point // If the node is nil, then it's a child of the fork point
// fullnode(it's an non-existent branch). // fullnode(it's a non-existent branch).
if removeLeft {
return errors.New("invalid right edge proof")
}
return nil return nil
default: default:
panic("it shouldn't happen") // hashNode, valueNode panic("it shouldn't happen") // hashNode, valueNode
@ -380,34 +424,37 @@ func hasRightElement(node node, key []byte) bool {
return false return false
} }
// VerifyRangeProof checks whether the given leaf nodes and edge proofs // VerifyRangeProof checks whether the given leaf nodes and edge proof
// can prove the given trie leaves range is matched with given root hash // can prove the given trie leaves range is matched with the specific root.
// and the range is consecutive(no gap inside) and monotonic increasing. // Besides, the range should be consecutive(no gap inside) and monotonic
// increasing.
// //
// Note the given first edge proof can be non-existing proof. For example // Note the given proof actually contains two edge proofs. Both of them can
// the first proof is for an non-existent values 0x03. The given batch // be non-existent proofs. For example the first proof is for a non-existent
// leaves are [0x04, 0x05, .. 0x09]. It's still feasible to prove. But the // key 0x03, the last proof is for a non-existent key 0x10. The given batch
// last edge proof should always be an existent proof. // leaves are [0x04, 0x05, .. 0x09]. It's still feasible to prove the given
// batch is valid.
// //
// The firstKey is paired with firstProof, not necessarily the same as keys[0] // The firstKey is paired with firstProof, not necessarily the same as keys[0]
// (unless firstProof is an existent proof). // (unless firstProof is an existent proof). Similarly, lastKey and lastProof
// are paired.
// //
// Expect the normal case, this function can also be used to verify the following // Expect the normal case, this function can also be used to verify the following
// range proofs: // range proofs:
// //
// - All elements proof. In this case the left and right proof can be nil, but the // - All elements proof. In this case the proof can be nil, but the range should
// range should be all the leaves in the trie. // be all the leaves in the trie.
// //
// - One element proof. In this case no matter the left edge proof is a non-existent // - One element proof. In this case no matter the edge proof is a non-existent
// proof or not, we can always verify the correctness of the proof. // proof or not, we can always verify the correctness of the proof.
// //
// - Zero element proof(left edge proof should be a non-existent proof). In this // - Zero element proof. In this case a single non-existent proof is enough to prove.
// case if there are still some other leaves available on the right side, then // Besides, if there are still some other leaves available on the right side, then
// an error will be returned. // an error will be returned.
// //
// Except returning the error to indicate the proof is valid or not, the function will // Except returning the error to indicate the proof is valid or not, the function will
// also return a flag to indicate whether there exists more accounts/slots in the trie. // also return a flag to indicate whether there exists more accounts/slots in the trie.
func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, values [][]byte, firstProof ethdb.KeyValueReader, lastProof ethdb.KeyValueReader) (error, bool) { func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (error, bool) {
if len(keys) != len(values) { if len(keys) != len(values) {
return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false
} }
@ -419,7 +466,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu
} }
// Special case, there is no edge proof at all. The given range is expected // Special case, there is no edge proof at all. The given range is expected
// to be the whole leaf-set in the trie. // to be the whole leaf-set in the trie.
if firstProof == nil && lastProof == nil { if proof == nil {
emptytrie, err := New(common.Hash{}, NewDatabase(memorydb.New())) emptytrie, err := New(common.Hash{}, NewDatabase(memorydb.New()))
if err != nil { if err != nil {
return err, false return err, false
@ -432,10 +479,10 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu
} }
return nil, false // no more element. return nil, false // no more element.
} }
// Special case, there is a provided left edge proof and zero key/value // Special case, there is a provided edge proof but zero key/value
// pairs, ensure there are no more accounts / slots in the trie. // pairs, ensure there are no more accounts / slots in the trie.
if len(keys) == 0 { if len(keys) == 0 {
root, val, err := proofToPath(rootHash, nil, firstKey, firstProof, true) root, val, err := proofToPath(rootHash, nil, firstKey, proof, true)
if err != nil { if err != nil {
return err, false return err, false
} }
@ -444,35 +491,47 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu
} }
return nil, false return nil, false
} }
// Special case, there is only one element and left edge // Special case, there is only one element and two edge keys are same.
// proof is an existent one. // In this case, we can't construct two edge paths. So handle it here.
if len(keys) == 1 && bytes.Equal(keys[0], firstKey) { if len(keys) == 1 && bytes.Equal(firstKey, lastKey) {
root, val, err := proofToPath(rootHash, nil, firstKey, firstProof, false) root, val, err := proofToPath(rootHash, nil, firstKey, proof, false)
if err != nil { if err != nil {
return err, false return err, false
} }
if !bytes.Equal(val, values[0]) { if !bytes.Equal(firstKey, keys[0]) {
return fmt.Errorf("correct proof but invalid data"), false return errors.New("correct proof but invalid key"), false
} }
return nil, hasRightElement(root, keys[0]) if !bytes.Equal(val, values[0]) {
return errors.New("correct proof but invalid data"), false
}
return nil, hasRightElement(root, firstKey)
}
// Ok, in all other cases, we require two edge paths available.
// First check the validity of edge keys.
if bytes.Compare(firstKey, lastKey) >= 0 {
return errors.New("invalid edge keys"), false
}
// todo(rjl493456442) different length edge keys should be supported
if len(firstKey) != len(lastKey) {
return errors.New("inconsistent edge keys"), false
} }
// Convert the edge proofs to edge trie paths. Then we can // Convert the edge proofs to edge trie paths. Then we can
// have the same tree architecture with the original one. // have the same tree architecture with the original one.
// For the first edge proof, non-existent proof is allowed. // For the first edge proof, non-existent proof is allowed.
root, _, err := proofToPath(rootHash, nil, firstKey, firstProof, true) root, _, err := proofToPath(rootHash, nil, firstKey, proof, true)
if err != nil { if err != nil {
return err, false return err, false
} }
// Pass the root node here, the second path will be merged // Pass the root node here, the second path will be merged
// with the first one. For the last edge proof, non-existent // with the first one. For the last edge proof, non-existent
// proof is not allowed. // proof is also allowed.
root, _, err = proofToPath(rootHash, root, keys[len(keys)-1], lastProof, false) root, _, err = proofToPath(rootHash, root, lastKey, proof, true)
if err != nil { if err != nil {
return err, false return err, false
} }
// Remove all internal references. All the removed parts should // Remove all internal references. All the removed parts should
// be re-filled(or re-constructed) by the given leaves range. // be re-filled(or re-constructed) by the given leaves range.
if err := unsetInternal(root, firstKey, keys[len(keys)-1]); err != nil { if err := unsetInternal(root, firstKey, lastKey); err != nil {
return err, false return err, false
} }
// Rebuild the trie with the leave stream, the shape of trie // Rebuild the trie with the leave stream, the shape of trie

352
trie/proof_test.go
View File

@ -166,15 +166,13 @@ func TestRangeProof(t *testing.T) {
sort.Sort(entries) sort.Sort(entries)
for i := 0; i < 500; i++ { for i := 0; i < 500; i++ {
start := mrand.Intn(len(entries)) start := mrand.Intn(len(entries))
end := mrand.Intn(len(entries)-start) + start end := mrand.Intn(len(entries)-start) + start + 1
if start == end {
continue proof := memorydb.New()
} if err := trie.Prove(entries[start].k, 0, proof); err != nil {
firstProof, lastProof := memorydb.New(), memorydb.New()
if err := trie.Prove(entries[start].k, 0, firstProof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { if err := trie.Prove(entries[end-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
var keys [][]byte var keys [][]byte
@ -183,15 +181,15 @@ func TestRangeProof(t *testing.T) {
keys = append(keys, entries[i].k) keys = append(keys, entries[i].k)
vals = append(vals, entries[i].v) vals = append(vals, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
if err != nil { if err != nil {
t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
} }
} }
} }
// TestRangeProof tests normal range proof with the first edge proof // TestRangeProof tests normal range proof with two non-existent proofs.
// as the non-existent proof. The test cases are generated randomly. // The test cases are generated randomly.
func TestRangeProofWithNonExistentProof(t *testing.T) { func TestRangeProofWithNonExistentProof(t *testing.T) {
trie, vals := randomTrie(4096) trie, vals := randomTrie(4096)
var entries entrySlice var entries entrySlice
@ -201,20 +199,31 @@ func TestRangeProofWithNonExistentProof(t *testing.T) {
sort.Sort(entries) sort.Sort(entries)
for i := 0; i < 500; i++ { for i := 0; i < 500; i++ {
start := mrand.Intn(len(entries)) start := mrand.Intn(len(entries))
end := mrand.Intn(len(entries)-start) + start end := mrand.Intn(len(entries)-start) + start + 1
if start == end { proof := memorydb.New()
continue
}
firstProof, lastProof := memorydb.New(), memorydb.New()
// Short circuit if the decreased key is same with the previous key
first := decreseKey(common.CopyBytes(entries[start].k)) first := decreseKey(common.CopyBytes(entries[start].k))
if start != 0 && bytes.Equal(first, entries[start-1].k) { if start != 0 && bytes.Equal(first, entries[start-1].k) {
continue continue
} }
if err := trie.Prove(first, 0, firstProof); err != nil { // Short circuit if the decreased key is underflow
if bytes.Compare(first, entries[start].k) > 0 {
continue
}
// Short circuit if the increased key is same with the next key
last := increseKey(common.CopyBytes(entries[end-1].k))
if end != len(entries) && bytes.Equal(last, entries[end].k) {
continue
}
// Short circuit if the increased key is overflow
if bytes.Compare(last, entries[end-1].k) < 0 {
continue
}
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
var keys [][]byte var keys [][]byte
@ -223,16 +232,36 @@ func TestRangeProofWithNonExistentProof(t *testing.T) {
keys = append(keys, entries[i].k) keys = append(keys, entries[i].k)
vals = append(vals, entries[i].v) vals = append(vals, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), first, keys, vals, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
if err != nil { if err != nil {
t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
} }
} }
// Special case, two edge proofs for two edge key.
proof := memorydb.New()
first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes()
last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
var k [][]byte
var v [][]byte
for i := 0; i < len(entries); i++ {
k = append(k, entries[i].k)
v = append(v, entries[i].v)
}
err, _ := VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
if err != nil {
t.Fatal("Failed to verify whole rang with non-existent edges")
}
} }
// TestRangeProofWithInvalidNonExistentProof tests such scenarios: // TestRangeProofWithInvalidNonExistentProof tests such scenarios:
// - The last edge proof is an non-existent proof
// - There exists a gap between the first element and the left edge proof // - There exists a gap between the first element and the left edge proof
// - There exists a gap between the last element and the right edge proof
func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { func TestRangeProofWithInvalidNonExistentProof(t *testing.T) {
trie, vals := randomTrie(4096) trie, vals := randomTrie(4096)
var entries entrySlice var entries entrySlice
@ -243,44 +272,45 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) {
// Case 1 // Case 1
start, end := 100, 200 start, end := 100, 200
first, last := decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[end].k)) first := decreseKey(common.CopyBytes(entries[start].k))
firstProof, lastProof := memorydb.New(), memorydb.New()
if err := trie.Prove(first, 0, firstProof); err != nil { proof := memorydb.New()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(last, 0, lastProof); err != nil { if err := trie.Prove(entries[end-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
var k [][]byte start = 105 // Gap created
var v [][]byte k := make([][]byte, 0)
v := make([][]byte, 0)
for i := start; i < end; i++ { for i := start; i < end; i++ {
k = append(k, entries[i].k) k = append(k, entries[i].k)
v = append(v, entries[i].v) v = append(v, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof)
if err == nil { if err == nil {
t.Fatalf("Expected to detect the error, got nil") t.Fatalf("Expected to detect the error, got nil")
} }
// Case 2 // Case 2
start, end = 100, 200 start, end = 100, 200
first = decreseKey(common.CopyBytes(entries[start].k)) last := increseKey(common.CopyBytes(entries[end-1].k))
proof = memorydb.New()
firstProof, lastProof = memorydb.New(), memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil {
if err := trie.Prove(first, 0, firstProof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
start = 105 // Gap created end = 195 // Capped slice
k = make([][]byte, 0) k = make([][]byte, 0)
v = make([][]byte, 0) v = make([][]byte, 0)
for i := start; i < end; i++ { for i := start; i < end; i++ {
k = append(k, entries[i].k) k = append(k, entries[i].k)
v = append(v, entries[i].v) v = append(v, entries[i].v)
} }
err, _ = VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) err, _ = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof)
if err == nil { if err == nil {
t.Fatalf("Expected to detect the error, got nil") t.Fatalf("Expected to detect the error, got nil")
} }
@ -297,31 +327,59 @@ func TestOneElementRangeProof(t *testing.T) {
} }
sort.Sort(entries) sort.Sort(entries)
// One element with existent edge proof // One element with existent edge proof, both edge proofs
// point to the SAME key.
start := 1000 start := 1000
firstProof, lastProof := memorydb.New(), memorydb.New() proof := memorydb.New()
if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { if err := trie.Prove(entries[start].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
// One element with non-existent edge proof // One element with left non-existent edge proof
start = 1000 start = 1000
first := decreseKey(common.CopyBytes(entries[start].k)) first := decreseKey(common.CopyBytes(entries[start].k))
firstProof, lastProof = memorydb.New(), memorydb.New() proof = memorydb.New()
if err := trie.Prove(first, 0, firstProof); err != nil { if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { if err := trie.Prove(entries[start].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
err, _ = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof) err, _ = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// One element with right non-existent edge proof
start = 1000
last := increseKey(common.CopyBytes(entries[start].k))
proof = memorydb.New()
if err := trie.Prove(entries[start].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// One element with two non-existent edge proofs
start = 1000
first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k))
proof = memorydb.New()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
@ -343,20 +401,35 @@ func TestAllElementsProof(t *testing.T) {
k = append(k, entries[i].k) k = append(k, entries[i].k)
v = append(v, entries[i].v) v = append(v, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), k[0], k, v, nil, nil) err, _ := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
// Even with edge proofs, it should still work. // With edge proofs, it should still work.
firstProof, lastProof := memorydb.New(), memorydb.New() proof := memorydb.New()
if err := trie.Prove(entries[0].k, 0, firstProof); err != nil { if err := trie.Prove(entries[0].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[len(entries)-1].k, 0, lastProof); err != nil { if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
err, _ = VerifyRangeProof(trie.Hash(), k[0], k, v, firstProof, lastProof) err, _ = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
// Even with non-existent edge proofs, it should still work.
proof = memorydb.New()
first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes()
last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ = VerifyRangeProof(trie.Hash(), first, last, k, v, proof)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
@ -376,11 +449,11 @@ func TestSingleSideRangeProof(t *testing.T) {
var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1}
for _, pos := range cases { for _, pos := range cases {
firstProof, lastProof := memorydb.New(), memorydb.New() proof := memorydb.New()
if err := trie.Prove(common.Hash{}.Bytes(), 0, firstProof); err != nil { if err := trie.Prove(common.Hash{}.Bytes(), 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[pos].k, 0, lastProof); err != nil { if err := trie.Prove(entries[pos].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
k := make([][]byte, 0) k := make([][]byte, 0)
@ -389,7 +462,43 @@ func TestSingleSideRangeProof(t *testing.T) {
k = append(k, entries[i].k) k = append(k, entries[i].k)
v = append(v, entries[i].v) v = append(v, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k, v, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof)
if err != nil {
t.Fatalf("Expected no error, got %v", err)
}
}
}
}
// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff.
func TestReverseSingleSideRangeProof(t *testing.T) {
for i := 0; i < 64; i++ {
trie := new(Trie)
var entries entrySlice
for i := 0; i < 4096; i++ {
value := &kv{randBytes(32), randBytes(20), false}
trie.Update(value.k, value.v)
entries = append(entries, value)
}
sort.Sort(entries)
var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1}
for _, pos := range cases {
proof := memorydb.New()
if err := trie.Prove(entries[pos].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")
if err := trie.Prove(last.Bytes(), 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
k := make([][]byte, 0)
v := make([][]byte, 0)
for i := pos; i < len(entries); i++ {
k = append(k, entries[i].k)
v = append(v, entries[i].v)
}
err, _ := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
@ -409,15 +518,12 @@ func TestBadRangeProof(t *testing.T) {
for i := 0; i < 500; i++ { for i := 0; i < 500; i++ {
start := mrand.Intn(len(entries)) start := mrand.Intn(len(entries))
end := mrand.Intn(len(entries)-start) + start end := mrand.Intn(len(entries)-start) + start + 1
if start == end { proof := memorydb.New()
continue if err := trie.Prove(entries[start].k, 0, proof); err != nil {
}
firstProof, lastProof := memorydb.New(), memorydb.New()
if err := trie.Prove(entries[start].k, 0, firstProof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { if err := trie.Prove(entries[end-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
var keys [][]byte var keys [][]byte
@ -426,6 +532,7 @@ func TestBadRangeProof(t *testing.T) {
keys = append(keys, entries[i].k) keys = append(keys, entries[i].k)
vals = append(vals, entries[i].v) vals = append(vals, entries[i].v)
} }
var first, last = keys[0], keys[len(keys)-1]
testcase := mrand.Intn(6) testcase := mrand.Intn(6)
var index int var index int
switch testcase { switch testcase {
@ -439,17 +546,6 @@ func TestBadRangeProof(t *testing.T) {
vals[index] = randBytes(20) // In theory it can't be same vals[index] = randBytes(20) // In theory it can't be same
case 2: case 2:
// Gapped entry slice // Gapped entry slice
// There are only two elements, skip it. Dropped any element
// will lead to single edge proof which is always correct.
if end-start <= 2 {
continue
}
// If the dropped element is the first or last one and it's a
// batch of small size elements. In this special case, it can
// happen that the proof for the edge element is exactly same
// with the first/last second element(since small values are
// embedded in the parent). Avoid this case.
index = mrand.Intn(end - start) index = mrand.Intn(end - start)
if (index == 0 && start < 100) || (index == end-start-1 && end <= 100) { if (index == 0 && start < 100) || (index == end-start-1 && end <= 100) {
continue continue
@ -457,20 +553,24 @@ func TestBadRangeProof(t *testing.T) {
keys = append(keys[:index], keys[index+1:]...) keys = append(keys[:index], keys[index+1:]...)
vals = append(vals[:index], vals[index+1:]...) vals = append(vals[:index], vals[index+1:]...)
case 3: case 3:
// Switched entry slice, same effect with gapped // Out of order
index = mrand.Intn(end - start) index1 := mrand.Intn(end - start)
keys[index] = entries[len(entries)-1].k index2 := mrand.Intn(end - start)
vals[index] = entries[len(entries)-1].v if index1 == index2 {
continue
}
keys[index1], keys[index2] = keys[index2], keys[index1]
vals[index1], vals[index2] = vals[index2], vals[index1]
case 4: case 4:
// Set random key to nil // Set random key to nil, do nothing
index = mrand.Intn(end - start) index = mrand.Intn(end - start)
keys[index] = nil keys[index] = nil
case 5: case 5:
// Set random value to nil // Set random value to nil, deletion
index = mrand.Intn(end - start) index = mrand.Intn(end - start)
vals[index] = nil vals[index] = nil
} }
err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof)
if err == nil { if err == nil {
t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1)
} }
@ -488,11 +588,11 @@ func TestGappedRangeProof(t *testing.T) {
entries = append(entries, value) entries = append(entries, value)
} }
first, last := 2, 8 first, last := 2, 8
firstProof, lastProof := memorydb.New(), memorydb.New() proof := memorydb.New()
if err := trie.Prove(entries[first].k, 0, firstProof); err != nil { if err := trie.Prove(entries[first].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[last-1].k, 0, lastProof); err != nil { if err := trie.Prove(entries[last-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err) t.Fatalf("Failed to prove the last node %v", err)
} }
var keys [][]byte var keys [][]byte
@ -504,12 +604,55 @@ func TestGappedRangeProof(t *testing.T) {
keys = append(keys, entries[i].k) keys = append(keys, entries[i].k)
vals = append(vals, entries[i].v) vals = append(vals, entries[i].v)
} }
err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof)
if err == nil { if err == nil {
t.Fatal("expect error, got nil") t.Fatal("expect error, got nil")
} }
} }
// TestSameSideProofs tests the element is not in the range covered by proofs
func TestSameSideProofs(t *testing.T) {
trie, vals := randomTrie(4096)
var entries entrySlice
for _, kv := range vals {
entries = append(entries, kv)
}
sort.Sort(entries)
pos := 1000
first := decreseKey(common.CopyBytes(entries[pos].k))
first = decreseKey(first)
last := decreseKey(common.CopyBytes(entries[pos].k))
proof := memorydb.New()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
if err == nil {
t.Fatalf("Expected error, got nil")
}
first = increseKey(common.CopyBytes(entries[pos].k))
last = increseKey(common.CopyBytes(entries[pos].k))
last = increseKey(last)
proof = memorydb.New()
if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
if err := trie.Prove(last, 0, proof); err != nil {
t.Fatalf("Failed to prove the last node %v", err)
}
err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof)
if err == nil {
t.Fatalf("Expected error, got nil")
}
}
func TestHasRightElement(t *testing.T) { func TestHasRightElement(t *testing.T) {
trie := new(Trie) trie := new(Trie)
var entries entrySlice var entries entrySlice
@ -530,38 +673,49 @@ func TestHasRightElement(t *testing.T) {
{0, 10, true}, {0, 10, true},
{50, 100, true}, {50, 100, true},
{50, len(entries), false}, // No more element expected {50, len(entries), false}, // No more element expected
{len(entries) - 1, len(entries), false}, // Single last element {len(entries) - 1, len(entries), false}, // Single last element with two existent proofs(point to same key)
{len(entries) - 1, -1, false}, // Single last element with non-existent right proof
{0, len(entries), false}, // The whole set with existent left proof {0, len(entries), false}, // The whole set with existent left proof
{-1, len(entries), false}, // The whole set with non-existent left proof {-1, len(entries), false}, // The whole set with non-existent left proof
{-1, -1, false}, // The whole set with non-existent left/right proof
} }
for _, c := range cases { for _, c := range cases {
var ( var (
firstKey []byte firstKey []byte
lastKey []byte
start = c.start start = c.start
firstProof = memorydb.New() end = c.end
lastProof = memorydb.New() proof = memorydb.New()
) )
if c.start == -1 { if c.start == -1 {
firstKey, start = common.Hash{}.Bytes(), 0 firstKey, start = common.Hash{}.Bytes(), 0
if err := trie.Prove(firstKey, 0, firstProof); err != nil { if err := trie.Prove(firstKey, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
} else { } else {
firstKey = entries[c.start].k firstKey = entries[c.start].k
if err := trie.Prove(entries[c.start].k, 0, firstProof); err != nil { if err := trie.Prove(entries[c.start].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
} }
if err := trie.Prove(entries[c.end-1].k, 0, lastProof); err != nil { if c.end == -1 {
lastKey, end = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes(), len(entries)
if err := trie.Prove(lastKey, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
} else {
lastKey = entries[c.end-1].k
if err := trie.Prove(entries[c.end-1].k, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err)
}
}
k := make([][]byte, 0) k := make([][]byte, 0)
v := make([][]byte, 0) v := make([][]byte, 0)
for i := start; i < c.end; i++ { for i := start; i < end; i++ {
k = append(k, entries[i].k) k = append(k, entries[i].k)
v = append(v, entries[i].v) v = append(v, entries[i].v)
} }
err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, k, v, firstProof, lastProof) err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof)
if err != nil { if err != nil {
t.Fatalf("Expected no error, got %v", err) t.Fatalf("Expected no error, got %v", err)
} }
@ -589,12 +743,12 @@ func TestEmptyRangeProof(t *testing.T) {
{500, true}, {500, true},
} }
for _, c := range cases { for _, c := range cases {
firstProof := memorydb.New() proof := memorydb.New()
first := increseKey(common.CopyBytes(entries[c.pos].k)) first := increseKey(common.CopyBytes(entries[c.pos].k))
if err := trie.Prove(first, 0, firstProof); err != nil { if err := trie.Prove(first, 0, proof); err != nil {
t.Fatalf("Failed to prove the first node %v", err) t.Fatalf("Failed to prove the first node %v", err)
} }
err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, firstProof, nil) err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof)
if c.err && err == nil { if c.err && err == nil {
t.Fatalf("Expected error, got nil") t.Fatalf("Expected error, got nil")
} }
@ -688,11 +842,11 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) {
start := 2 start := 2
end := start + size end := start + size
firstProof, lastProof := memorydb.New(), memorydb.New() proof := memorydb.New()
if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { if err := trie.Prove(entries[start].k, 0, proof); err != nil {
b.Fatalf("Failed to prove the first node %v", err) b.Fatalf("Failed to prove the first node %v", err)
} }
if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { if err := trie.Prove(entries[end-1].k, 0, proof); err != nil {
b.Fatalf("Failed to prove the last node %v", err) b.Fatalf("Failed to prove the last node %v", err)
} }
var keys [][]byte var keys [][]byte
@ -704,7 +858,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) {
b.ResetTimer() b.ResetTimer()
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, values, firstProof, lastProof) err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof)
if err != nil { if err != nil {
b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err)
} }