From e5defccd581880101aa0897705e3e9c3735a8186 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 23 Sep 2020 17:44:09 +0800 Subject: [PATCH] trie: extend range proof (#21250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * trie: support non-existent right proof * trie: improve test * trie: minor linter fix Co-authored-by: Péter Szilágyi --- trie/proof.go | 223 +++++++++++++++++----------- trie/proof_test.go | 358 ++++++++++++++++++++++++++++++++------------- 2 files changed, 397 insertions(+), 184 deletions(-) diff --git a/trie/proof.go b/trie/proof.go index 326235a3f..2f52438f9 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -129,10 +129,11 @@ func VerifyProof(rootHash common.Hash, key []byte, proofDb ethdb.KeyValueReader) } } -// proofToPath converts a merkle proof to trie node path. -// The main purpose of this function is recovering a node -// path from the merkle proof stream. All necessary nodes -// will be resolved and leave the remaining as hashnode. +// proofToPath converts a merkle proof to trie node path. The main purpose of +// this function is recovering a node path from the merkle proof stream. All +// necessary nodes will be resolved and leave the remaining as hashnode. +// +// The given edge proof is allowed to be an existent or non-existent proof. func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyValueReader, allowNonExistent bool) (node, []byte, error) { // resolveNode retrieves and resolves trie node from merkle proof stream resolveNode := func(hash common.Hash) (node, error) { @@ -205,54 +206,61 @@ func proofToPath(rootHash common.Hash, root node, key []byte, proofDb ethdb.KeyV } // unsetInternal removes all internal node references(hashnode, embedded node). -// It should be called after a trie is constructed with two edge proofs. Also -// the given boundary keys must be the one used to construct the edge proofs. +// It should be called after a trie is constructed with two edge paths. Also +// the given boundary keys must be the one used to construct the edge paths. // // It's the key step for range proof. All visited nodes should be marked dirty // since the node content might be modified. Besides it can happen that some // fullnodes only have one child which is disallowed. But if the proof is valid, // the missing children will be filled, otherwise it will be thrown anyway. +// +// Note we have the assumption here the given boundary keys are different +// and right is larger than left. func unsetInternal(n node, left []byte, right []byte) error { left, right = keybytesToHex(left), keybytesToHex(right) - // todo(rjl493456442) different length edge keys should be supported - if len(left) != len(right) { - return errors.New("inconsistent edge path") - } // Step down to the fork point. There are two scenarios can happen: - // - the fork point is a shortnode: the left proof MUST point to a - // non-existent key and the key doesn't match with the shortnode - // - the fork point is a fullnode: the left proof can point to an - // existent key or not. + // - the fork point is a shortnode: either the key of left proof or + // right proof doesn't match with shortnode's key. + // - the fork point is a fullnode: both two edge proofs are allowed + // to point to a non-existent key. var ( pos = 0 parent node + + // fork indicator, 0 means no fork, -1 means proof is less, 1 means proof is greater + shortForkLeft, shortForkRight int ) findFork: for { switch rn := (n).(type) { case *shortNode: - // The right proof must point to an existent key. - if len(right)-pos < len(rn.Key) || !bytes.Equal(rn.Key, right[pos:pos+len(rn.Key)]) { - return errors.New("invalid edge path") - } rn.flags = nodeFlag{dirty: true} - // Special case, the non-existent proof points to the same path - // as the existent proof, but the path of existent proof is longer. - // In this case, the fork point is this shortnode. - if len(left)-pos < len(rn.Key) || !bytes.Equal(rn.Key, left[pos:pos+len(rn.Key)]) { + + // If either the key of left proof or right proof doesn't match with + // shortnode, stop here and the forkpoint is the shortnode. + if len(left)-pos < len(rn.Key) { + shortForkLeft = bytes.Compare(left[pos:], rn.Key) + } else { + shortForkLeft = bytes.Compare(left[pos:pos+len(rn.Key)], rn.Key) + } + if len(right)-pos < len(rn.Key) { + shortForkRight = bytes.Compare(right[pos:], rn.Key) + } else { + shortForkRight = bytes.Compare(right[pos:pos+len(rn.Key)], rn.Key) + } + if shortForkLeft != 0 || shortForkRight != 0 { break findFork } parent = n n, pos = rn.Val, pos+len(rn.Key) case *fullNode: - leftnode, rightnode := rn.Children[left[pos]], rn.Children[right[pos]] - // The right proof must point to an existent key. - if rightnode == nil { - return errors.New("invalid edge path") - } rn.flags = nodeFlag{dirty: true} - if leftnode != rightnode { + + // If either the node pointed by left proof or right proof is nil, + // stop here and the forkpoint is the fullnode. + leftnode, rightnode := rn.Children[left[pos]], rn.Children[right[pos]] + if leftnode == nil || rightnode == nil || leftnode != rightnode { break findFork } parent = n @@ -263,12 +271,42 @@ findFork: } switch rn := n.(type) { case *shortNode: - if _, ok := rn.Val.(valueNode); ok { - parent.(*fullNode).Children[right[pos-1]] = nil + // There can have these five scenarios: + // - both proofs are less than the trie path => no valid range + // - both proofs are greater than the trie path => no valid range + // - left proof is less and right proof is greater => valid range, unset the shortnode entirely + // - left proof points to the shortnode, but right proof is greater + // - right proof points to the shortnode, but left proof is less + if shortForkLeft == -1 && shortForkRight == -1 { + return errors.New("empty range") + } + if shortForkLeft == 1 && shortForkRight == 1 { + return errors.New("empty range") + } + if shortForkLeft != 0 && shortForkRight != 0 { + parent.(*fullNode).Children[left[pos-1]] = nil return nil } - return unset(rn, rn.Val, right[pos:], len(rn.Key), true) + // Only one proof points to non-existent key. + if shortForkRight != 0 { + // Unset left proof's path + if _, ok := rn.Val.(valueNode); ok { + parent.(*fullNode).Children[left[pos-1]] = nil + return nil + } + return unset(rn, rn.Val, left[pos:], len(rn.Key), false) + } + if shortForkLeft != 0 { + // Unset right proof's path. + if _, ok := rn.Val.(valueNode); ok { + parent.(*fullNode).Children[right[pos-1]] = nil + return nil + } + return unset(rn, rn.Val, right[pos:], len(rn.Key), true) + } + return nil case *fullNode: + // unset all internal nodes in the forkpoint for i := left[pos] + 1; i < right[pos]; i++ { rn.Children[i] = nil } @@ -285,19 +323,17 @@ findFork: } // unset removes all internal node references either the left most or right most. -// If we try to unset all right most references, it can meet these scenarios: +// It can meet these scenarios: // -// - The given path is existent in the trie, unset the associated shortnode +// - The given path is existent in the trie, unset the associated nodes with the +// specific direction // - The given path is non-existent in the trie // - the fork point is a fullnode, the corresponding child pointed by path // is nil, return -// - the fork point is a shortnode, the key of shortnode is less than path, +// - the fork point is a shortnode, the shortnode is included in the range, // keep the entire branch and return. -// - the fork point is a shortnode, the key of shortnode is greater than path, +// - the fork point is a shortnode, the shortnode is excluded in the range, // unset the entire branch. -// -// If we try to unset all left most references, then the given path should -// be existent. func unset(parent node, child node, key []byte, pos int, removeLeft bool) error { switch cld := child.(type) { case *fullNode: @@ -317,18 +353,29 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error if len(key[pos:]) < len(cld.Key) || !bytes.Equal(cld.Key, key[pos:pos+len(cld.Key)]) { // Find the fork point, it's an non-existent branch. if removeLeft { - return errors.New("invalid right edge proof") - } - if bytes.Compare(cld.Key, key[pos:]) > 0 { - // The key of fork shortnode is greater than the - // path(it belongs to the range), unset the entrie - // branch. The parent must be a fullnode. - fn := parent.(*fullNode) - fn.Children[key[pos-1]] = nil + if bytes.Compare(cld.Key, key[pos:]) < 0 { + // The key of fork shortnode is less than the path + // (it belongs to the range), unset the entrie + // branch. The parent must be a fullnode. + fn := parent.(*fullNode) + fn.Children[key[pos-1]] = nil + } else { + // The key of fork shortnode is greater than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + } } else { - // The key of fork shortnode is less than the - // path(it doesn't belong to the range), keep - // it with the cached hash available. + if bytes.Compare(cld.Key, key[pos:]) > 0 { + // The key of fork shortnode is greater than the + // path(it belongs to the range), unset the entrie + // branch. The parent must be a fullnode. + fn := parent.(*fullNode) + fn.Children[key[pos-1]] = nil + } else { + // The key of fork shortnode is less than the + // path(it doesn't belong to the range), keep + // it with the cached hash available. + } } return nil } @@ -340,11 +387,8 @@ func unset(parent node, child node, key []byte, pos int, removeLeft bool) error cld.flags = nodeFlag{dirty: true} return unset(cld, cld.Val, key, pos+len(cld.Key), removeLeft) case nil: - // If the node is nil, it's a child of the fork point - // fullnode(it's an non-existent branch). - if removeLeft { - return errors.New("invalid right edge proof") - } + // If the node is nil, then it's a child of the fork point + // fullnode(it's a non-existent branch). return nil default: panic("it shouldn't happen") // hashNode, valueNode @@ -380,34 +424,37 @@ func hasRightElement(node node, key []byte) bool { return false } -// VerifyRangeProof checks whether the given leaf nodes and edge proofs -// can prove the given trie leaves range is matched with given root hash -// and the range is consecutive(no gap inside) and monotonic increasing. +// VerifyRangeProof checks whether the given leaf nodes and edge proof +// can prove the given trie leaves range is matched with the specific root. +// Besides, the range should be consecutive(no gap inside) and monotonic +// increasing. // -// Note the given first edge proof can be non-existing proof. For example -// the first proof is for an non-existent values 0x03. The given batch -// leaves are [0x04, 0x05, .. 0x09]. It's still feasible to prove. But the -// last edge proof should always be an existent proof. +// Note the given proof actually contains two edge proofs. Both of them can +// be non-existent proofs. For example the first proof is for a non-existent +// key 0x03, the last proof is for a non-existent key 0x10. The given batch +// leaves are [0x04, 0x05, .. 0x09]. It's still feasible to prove the given +// batch is valid. // // The firstKey is paired with firstProof, not necessarily the same as keys[0] -// (unless firstProof is an existent proof). +// (unless firstProof is an existent proof). Similarly, lastKey and lastProof +// are paired. // // Expect the normal case, this function can also be used to verify the following // range proofs: // -// - All elements proof. In this case the left and right proof can be nil, but the -// range should be all the leaves in the trie. +// - All elements proof. In this case the proof can be nil, but the range should +// be all the leaves in the trie. // -// - One element proof. In this case no matter the left edge proof is a non-existent +// - One element proof. In this case no matter the edge proof is a non-existent // proof or not, we can always verify the correctness of the proof. // -// - Zero element proof(left edge proof should be a non-existent proof). In this -// case if there are still some other leaves available on the right side, then +// - Zero element proof. In this case a single non-existent proof is enough to prove. +// Besides, if there are still some other leaves available on the right side, then // an error will be returned. // // Except returning the error to indicate the proof is valid or not, the function will // also return a flag to indicate whether there exists more accounts/slots in the trie. -func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, values [][]byte, firstProof ethdb.KeyValueReader, lastProof ethdb.KeyValueReader) (error, bool) { +func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, keys [][]byte, values [][]byte, proof ethdb.KeyValueReader) (error, bool) { if len(keys) != len(values) { return fmt.Errorf("inconsistent proof data, keys: %d, values: %d", len(keys), len(values)), false } @@ -419,7 +466,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu } // Special case, there is no edge proof at all. The given range is expected // to be the whole leaf-set in the trie. - if firstProof == nil && lastProof == nil { + if proof == nil { emptytrie, err := New(common.Hash{}, NewDatabase(memorydb.New())) if err != nil { return err, false @@ -432,10 +479,10 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu } return nil, false // no more element. } - // Special case, there is a provided left edge proof and zero key/value + // Special case, there is a provided edge proof but zero key/value // pairs, ensure there are no more accounts / slots in the trie. if len(keys) == 0 { - root, val, err := proofToPath(rootHash, nil, firstKey, firstProof, true) + root, val, err := proofToPath(rootHash, nil, firstKey, proof, true) if err != nil { return err, false } @@ -444,35 +491,47 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, keys [][]byte, valu } return nil, false } - // Special case, there is only one element and left edge - // proof is an existent one. - if len(keys) == 1 && bytes.Equal(keys[0], firstKey) { - root, val, err := proofToPath(rootHash, nil, firstKey, firstProof, false) + // Special case, there is only one element and two edge keys are same. + // In this case, we can't construct two edge paths. So handle it here. + if len(keys) == 1 && bytes.Equal(firstKey, lastKey) { + root, val, err := proofToPath(rootHash, nil, firstKey, proof, false) if err != nil { return err, false } - if !bytes.Equal(val, values[0]) { - return fmt.Errorf("correct proof but invalid data"), false + if !bytes.Equal(firstKey, keys[0]) { + return errors.New("correct proof but invalid key"), false } - return nil, hasRightElement(root, keys[0]) + if !bytes.Equal(val, values[0]) { + return errors.New("correct proof but invalid data"), false + } + return nil, hasRightElement(root, firstKey) + } + // Ok, in all other cases, we require two edge paths available. + // First check the validity of edge keys. + if bytes.Compare(firstKey, lastKey) >= 0 { + return errors.New("invalid edge keys"), false + } + // todo(rjl493456442) different length edge keys should be supported + if len(firstKey) != len(lastKey) { + return errors.New("inconsistent edge keys"), false } // Convert the edge proofs to edge trie paths. Then we can // have the same tree architecture with the original one. // For the first edge proof, non-existent proof is allowed. - root, _, err := proofToPath(rootHash, nil, firstKey, firstProof, true) + root, _, err := proofToPath(rootHash, nil, firstKey, proof, true) if err != nil { return err, false } // Pass the root node here, the second path will be merged // with the first one. For the last edge proof, non-existent - // proof is not allowed. - root, _, err = proofToPath(rootHash, root, keys[len(keys)-1], lastProof, false) + // proof is also allowed. + root, _, err = proofToPath(rootHash, root, lastKey, proof, true) if err != nil { return err, false } // Remove all internal references. All the removed parts should // be re-filled(or re-constructed) by the given leaves range. - if err := unsetInternal(root, firstKey, keys[len(keys)-1]); err != nil { + if err := unsetInternal(root, firstKey, lastKey); err != nil { return err, false } // Rebuild the trie with the leave stream, the shape of trie diff --git a/trie/proof_test.go b/trie/proof_test.go index 4d76f3f78..6cdc242d9 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -166,15 +166,13 @@ func TestRangeProof(t *testing.T) { sort.Sort(entries) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) - end := mrand.Intn(len(entries)-start) + start - if start == end { - continue - } - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { + end := mrand.Intn(len(entries)-start) + start + 1 + + proof := memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -183,15 +181,15 @@ func TestRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } } } -// TestRangeProof tests normal range proof with the first edge proof -// as the non-existent proof. The test cases are generated randomly. +// TestRangeProof tests normal range proof with two non-existent proofs. +// The test cases are generated randomly. func TestRangeProofWithNonExistentProof(t *testing.T) { trie, vals := randomTrie(4096) var entries entrySlice @@ -201,20 +199,31 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { sort.Sort(entries) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) - end := mrand.Intn(len(entries)-start) + start - if start == end { - continue - } - firstProof, lastProof := memorydb.New(), memorydb.New() + end := mrand.Intn(len(entries)-start) + start + 1 + proof := memorydb.New() + // Short circuit if the decreased key is same with the previous key first := decreseKey(common.CopyBytes(entries[start].k)) if start != 0 && bytes.Equal(first, entries[start-1].k) { continue } - if err := trie.Prove(first, 0, firstProof); err != nil { + // Short circuit if the decreased key is underflow + if bytes.Compare(first, entries[start].k) > 0 { + continue + } + // Short circuit if the increased key is same with the next key + last := increseKey(common.CopyBytes(entries[end-1].k)) + if end != len(entries) && bytes.Equal(last, entries[end].k) { + continue + } + // Short circuit if the increased key is overflow + if bytes.Compare(last, entries[end-1].k) < 0 { + continue + } + if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -223,16 +232,36 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), first, keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err != nil { t.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) } } + // Special case, two edge proofs for two edge key. + proof := memorydb.New() + first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() + last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() + if err := trie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + var k [][]byte + var v [][]byte + for i := 0; i < len(entries); i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), first, last, k, v, proof) + if err != nil { + t.Fatal("Failed to verify whole rang with non-existent edges") + } } // TestRangeProofWithInvalidNonExistentProof tests such scenarios: -// - The last edge proof is an non-existent proof // - There exists a gap between the first element and the left edge proof +// - There exists a gap between the last element and the right edge proof func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { trie, vals := randomTrie(4096) var entries entrySlice @@ -243,44 +272,45 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 1 start, end := 100, 200 - first, last := decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[end].k)) - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(first, 0, firstProof); err != nil { + first := decreseKey(common.CopyBytes(entries[start].k)) + + proof := memorydb.New() + if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, lastProof); err != nil { + if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - var k [][]byte - var v [][]byte + start = 105 // Gap created + k := make([][]byte, 0) + v := make([][]byte, 0) for i := start; i < end; i++ { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), first, k[len(k)-1], k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } // Case 2 start, end = 100, 200 - first = decreseKey(common.CopyBytes(entries[start].k)) - - firstProof, lastProof = memorydb.New(), memorydb.New() - if err := trie.Prove(first, 0, firstProof); err != nil { + last := increseKey(common.CopyBytes(entries[end-1].k)) + proof = memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + if err := trie.Prove(last, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - start = 105 // Gap created + end = 195 // Capped slice k = make([][]byte, 0) v = make([][]byte, 0) for i := start; i < end; i++ { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ = VerifyRangeProof(trie.Hash(), first, k, v, firstProof, lastProof) + err, _ = VerifyRangeProof(trie.Hash(), k[0], last, k, v, proof) if err == nil { t.Fatalf("Expected to detect the error, got nil") } @@ -297,31 +327,59 @@ func TestOneElementRangeProof(t *testing.T) { } sort.Sort(entries) - // One element with existent edge proof + // One element with existent edge proof, both edge proofs + // point to the SAME key. start := 1000 - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { + proof := memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { - t.Fatalf("Failed to prove the last node %v", err) - } - err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } - // One element with non-existent edge proof + // One element with left non-existent edge proof start = 1000 first := decreseKey(common.CopyBytes(entries[start].k)) - firstProof, lastProof = memorydb.New(), memorydb.New() - if err := trie.Prove(first, 0, firstProof); err != nil { + proof = memorydb.New() + if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[start].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), first, [][]byte{entries[start].k}, [][]byte{entries[start].v}, firstProof, lastProof) + err, _ = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + // One element with right non-existent edge proof + start = 1000 + last := increseKey(common.CopyBytes(entries[start].k)) + proof = memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + // One element with two non-existent edge proofs + start = 1000 + first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k)) + proof = memorydb.New() + if err := trie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -343,20 +401,35 @@ func TestAllElementsProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), k[0], k, v, nil, nil) + err, _ := VerifyRangeProof(trie.Hash(), nil, nil, k, v, nil) if err != nil { t.Fatalf("Expected no error, got %v", err) } - // Even with edge proofs, it should still work. - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[0].k, 0, firstProof); err != nil { + // With edge proofs, it should still work. + proof := memorydb.New() + if err := trie.Prove(entries[0].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[len(entries)-1].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } - err, _ = VerifyRangeProof(trie.Hash(), k[0], k, v, firstProof, lastProof) + err, _ = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + + // Even with non-existent edge proofs, it should still work. + proof = memorydb.New() + first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() + last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() + if err := trie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -376,11 +449,11 @@ func TestSingleSideRangeProof(t *testing.T) { var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(common.Hash{}.Bytes(), 0, firstProof); err != nil { + proof := memorydb.New() + if err := trie.Prove(common.Hash{}.Bytes(), 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[pos].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[pos].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } k := make([][]byte, 0) @@ -389,7 +462,43 @@ func TestSingleSideRangeProof(t *testing.T) { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k, v, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), common.Hash{}.Bytes(), k[len(k)-1], k, v, proof) + if err != nil { + t.Fatalf("Expected no error, got %v", err) + } + } + } +} + +// TestReverseSingleSideRangeProof tests the range ends with 0xffff...fff. +func TestReverseSingleSideRangeProof(t *testing.T) { + for i := 0; i < 64; i++ { + trie := new(Trie) + var entries entrySlice + for i := 0; i < 4096; i++ { + value := &kv{randBytes(32), randBytes(20), false} + trie.Update(value.k, value.v) + entries = append(entries, value) + } + sort.Sort(entries) + + var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} + for _, pos := range cases { + proof := memorydb.New() + if err := trie.Prove(entries[pos].k, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") + if err := trie.Prove(last.Bytes(), 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + k := make([][]byte, 0) + v := make([][]byte, 0) + for i := pos; i < len(entries); i++ { + k = append(k, entries[i].k) + v = append(v, entries[i].v) + } + err, _ := VerifyRangeProof(trie.Hash(), k[0], last.Bytes(), k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -409,15 +518,12 @@ func TestBadRangeProof(t *testing.T) { for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) - end := mrand.Intn(len(entries)-start) + start - if start == end { - continue - } - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { + end := mrand.Intn(len(entries)-start) + start + 1 + proof := memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -426,6 +532,7 @@ func TestBadRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } + var first, last = keys[0], keys[len(keys)-1] testcase := mrand.Intn(6) var index int switch testcase { @@ -439,17 +546,6 @@ func TestBadRangeProof(t *testing.T) { vals[index] = randBytes(20) // In theory it can't be same case 2: // Gapped entry slice - - // There are only two elements, skip it. Dropped any element - // will lead to single edge proof which is always correct. - if end-start <= 2 { - continue - } - // If the dropped element is the first or last one and it's a - // batch of small size elements. In this special case, it can - // happen that the proof for the edge element is exactly same - // with the first/last second element(since small values are - // embedded in the parent). Avoid this case. index = mrand.Intn(end - start) if (index == 0 && start < 100) || (index == end-start-1 && end <= 100) { continue @@ -457,20 +553,24 @@ func TestBadRangeProof(t *testing.T) { keys = append(keys[:index], keys[index+1:]...) vals = append(vals[:index], vals[index+1:]...) case 3: - // Switched entry slice, same effect with gapped - index = mrand.Intn(end - start) - keys[index] = entries[len(entries)-1].k - vals[index] = entries[len(entries)-1].v + // Out of order + index1 := mrand.Intn(end - start) + index2 := mrand.Intn(end - start) + if index1 == index2 { + continue + } + keys[index1], keys[index2] = keys[index2], keys[index1] + vals[index1], vals[index2] = vals[index2], vals[index1] case 4: - // Set random key to nil + // Set random key to nil, do nothing index = mrand.Intn(end - start) keys[index] = nil case 5: - // Set random value to nil + // Set random value to nil, deletion index = mrand.Intn(end - start) vals[index] = nil } - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), first, last, keys, vals, proof) if err == nil { t.Fatalf("%d Case %d index %d range: (%d->%d) expect error, got nil", i, testcase, index, start, end-1) } @@ -488,11 +588,11 @@ func TestGappedRangeProof(t *testing.T) { entries = append(entries, value) } first, last := 2, 8 - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[first].k, 0, firstProof); err != nil { + proof := memorydb.New() + if err := trie.Prove(entries[first].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[last-1].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[last-1].k, 0, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -504,12 +604,55 @@ func TestGappedRangeProof(t *testing.T) { keys = append(keys, entries[i].k) vals = append(vals, entries[i].v) } - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, vals, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof) if err == nil { t.Fatal("expect error, got nil") } } +// TestSameSideProofs tests the element is not in the range covered by proofs +func TestSameSideProofs(t *testing.T) { + trie, vals := randomTrie(4096) + var entries entrySlice + for _, kv := range vals { + entries = append(entries, kv) + } + sort.Sort(entries) + + pos := 1000 + first := decreseKey(common.CopyBytes(entries[pos].k)) + first = decreseKey(first) + last := decreseKey(common.CopyBytes(entries[pos].k)) + + proof := memorydb.New() + if err := trie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + if err == nil { + t.Fatalf("Expected error, got nil") + } + + first = increseKey(common.CopyBytes(entries[pos].k)) + last = increseKey(common.CopyBytes(entries[pos].k)) + last = increseKey(last) + + proof = memorydb.New() + if err := trie.Prove(first, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + if err := trie.Prove(last, 0, proof); err != nil { + t.Fatalf("Failed to prove the last node %v", err) + } + err, _ = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) + if err == nil { + t.Fatalf("Expected error, got nil") + } +} + func TestHasRightElement(t *testing.T) { trie := new(Trie) var entries entrySlice @@ -530,38 +673,49 @@ func TestHasRightElement(t *testing.T) { {0, 10, true}, {50, 100, true}, {50, len(entries), false}, // No more element expected - {len(entries) - 1, len(entries), false}, // Single last element + {len(entries) - 1, len(entries), false}, // Single last element with two existent proofs(point to same key) + {len(entries) - 1, -1, false}, // Single last element with non-existent right proof {0, len(entries), false}, // The whole set with existent left proof {-1, len(entries), false}, // The whole set with non-existent left proof + {-1, -1, false}, // The whole set with non-existent left/right proof } for _, c := range cases { var ( - firstKey []byte - start = c.start - firstProof = memorydb.New() - lastProof = memorydb.New() + firstKey []byte + lastKey []byte + start = c.start + end = c.end + proof = memorydb.New() ) if c.start == -1 { firstKey, start = common.Hash{}.Bytes(), 0 - if err := trie.Prove(firstKey, 0, firstProof); err != nil { + if err := trie.Prove(firstKey, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } else { firstKey = entries[c.start].k - if err := trie.Prove(entries[c.start].k, 0, firstProof); err != nil { + if err := trie.Prove(entries[c.start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } - if err := trie.Prove(entries[c.end-1].k, 0, lastProof); err != nil { - t.Fatalf("Failed to prove the first node %v", err) + if c.end == -1 { + lastKey, end = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes(), len(entries) + if err := trie.Prove(lastKey, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } + } else { + lastKey = entries[c.end-1].k + if err := trie.Prove(entries[c.end-1].k, 0, proof); err != nil { + t.Fatalf("Failed to prove the first node %v", err) + } } k := make([][]byte, 0) v := make([][]byte, 0) - for i := start; i < c.end; i++ { + for i := start; i < end; i++ { k = append(k, entries[i].k) v = append(v, entries[i].v) } - err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, k, v, firstProof, lastProof) + err, hasMore := VerifyRangeProof(trie.Hash(), firstKey, lastKey, k, v, proof) if err != nil { t.Fatalf("Expected no error, got %v", err) } @@ -589,12 +743,12 @@ func TestEmptyRangeProof(t *testing.T) { {500, true}, } for _, c := range cases { - firstProof := memorydb.New() + proof := memorydb.New() first := increseKey(common.CopyBytes(entries[c.pos].k)) - if err := trie.Prove(first, 0, firstProof); err != nil { + if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, firstProof, nil) + err, _ := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) if c.err && err == nil { t.Fatalf("Expected error, got nil") } @@ -688,11 +842,11 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { start := 2 end := start + size - firstProof, lastProof := memorydb.New(), memorydb.New() - if err := trie.Prove(entries[start].k, 0, firstProof); err != nil { + proof := memorydb.New() + if err := trie.Prove(entries[start].k, 0, proof); err != nil { b.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, lastProof); err != nil { + if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { b.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -704,7 +858,7 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { b.ResetTimer() for i := 0; i < b.N; i++ { - err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys, values, firstProof, lastProof) + err, _ := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, values, proof) if err != nil { b.Fatalf("Case %d(%d->%d) expect no error, got %v", i, start, end-1, err) }