tests/fuzzers: move fuzzers into native packages (#28467)

This PR moves our fuzzers from tests/fuzzers into whatever their respective 'native' package is. The historical reason why they were placed in an external location, is that when they were based on go-fuzz, they could not be "hidden" via the _test.go prefix. So in order to shove them away from the go-ethereum "production code", they were put aside. But now we've rewritten them to be based on golang testing, and thus can be brought back. I've left (in tests/) the ones that are not production (bls128381), require non-standard imports (secp requires btcec, bn256 requires gnark/google/cloudflare deps). This PR also adds a fuzzer for precompiled contracts, because why not. This PR utilizes a newly rewritten replacement for go-118-fuzz-build, namely gofuzz-shim, which utilises the inputs from the fuzzing engine better.
2023-11-14 14:34:29 +01:00
parent 24d46224c1
commit 2391fbc676
24 changed files with 700 additions and 992 deletions
@@ -0,0 +1,155 @@
+// Copyright 2020 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package trie
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/trie/trienode"
+	"golang.org/x/crypto/sha3"
+	"golang.org/x/exp/slices"
+)
+
+func FuzzStackTrie(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		fuzz(data, false)
+	})
+}
+
+func fuzz(data []byte, debugging bool) {
+	// This spongeDb is used to check the sequence of disk-db-writes
+	var (
+		input   = bytes.NewReader(data)
+		spongeA = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
+		dbA     = NewDatabase(rawdb.NewDatabase(spongeA), nil)
+		trieA   = NewEmpty(dbA)
+		spongeB = &spongeDb{sponge: sha3.NewLegacyKeccak256()}
+		dbB     = NewDatabase(rawdb.NewDatabase(spongeB), nil)
+
+		options = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) {
+			rawdb.WriteTrieNode(spongeB, common.Hash{}, path, hash, blob, dbB.Scheme())
+		})
+		trieB       = NewStackTrie(options)
+		vals        []*kv
+		maxElements = 10000
+		// operate on unique keys only
+		keys = make(map[string]struct{})
+	)
+	// Fill the trie with elements
+	for i := 0; input.Len() > 0 && i < maxElements; i++ {
+		k := make([]byte, 32)
+		input.Read(k)
+		var a uint16
+		binary.Read(input, binary.LittleEndian, &a)
+		a = 1 + a%100
+		v := make([]byte, a)
+		input.Read(v)
+		if input.Len() == 0 {
+			// If it was exhausted while reading, the value may be all zeroes,
+			// thus 'deletion' which is not supported on stacktrie
+			break
+		}
+		if _, present := keys[string(k)]; present {
+			// This key is a duplicate, ignore it
+			continue
+		}
+		keys[string(k)] = struct{}{}
+		vals = append(vals, &kv{k: k, v: v})
+		trieA.MustUpdate(k, v)
+	}
+	if len(vals) == 0 {
+		return
+	}
+	// Flush trie -> database
+	rootA, nodes, err := trieA.Commit(false)
+	if err != nil {
+		panic(err)
+	}
+	if nodes != nil {
+		dbA.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil)
+	}
+	// Flush memdb -> disk (sponge)
+	dbA.Commit(rootA, false)
+
+	// Stacktrie requires sorted insertion
+	slices.SortFunc(vals, (*kv).cmp)
+
+	for _, kv := range vals {
+		if debugging {
+			fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v)
+		}
+		trieB.MustUpdate(kv.k, kv.v)
+	}
+	rootB := trieB.Hash()
+	trieB.Commit()
+	if rootA != rootB {
+		panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootB))
+	}
+	sumA := spongeA.sponge.Sum(nil)
+	sumB := spongeB.sponge.Sum(nil)
+	if !bytes.Equal(sumA, sumB) {
+		panic(fmt.Sprintf("sequence differ: (trie) %x != %x (stacktrie)", sumA, sumB))
+	}
+
+	// Ensure all the nodes are persisted correctly
+	var (
+		nodeset  = make(map[string][]byte) // path -> blob
+		optionsC = NewStackTrieOptions().WithWriter(func(path []byte, hash common.Hash, blob []byte) {
+			if crypto.Keccak256Hash(blob) != hash {
+				panic("invalid node blob")
+			}
+			nodeset[string(path)] = common.CopyBytes(blob)
+		})
+		trieC   = NewStackTrie(optionsC)
+		checked int
+	)
+	for _, kv := range vals {
+		trieC.MustUpdate(kv.k, kv.v)
+	}
+	rootC := trieC.Commit()
+	if rootA != rootC {
+		panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootC))
+	}
+	trieA, _ = New(TrieID(rootA), dbA)
+	iterA := trieA.MustNodeIterator(nil)
+	for iterA.Next(true) {
+		if iterA.Hash() == (common.Hash{}) {
+			if _, present := nodeset[string(iterA.Path())]; present {
+				panic("unexpected tiny node")
+			}
+			continue
+		}
+		nodeBlob, present := nodeset[string(iterA.Path())]
+		if !present {
+			panic("missing node")
+		}
+		if !bytes.Equal(nodeBlob, iterA.NodeBlob()) {
+			panic("node blob is not matched")
+		}
+		checked += 1
+	}
+	if checked != len(nodeset) {
+		panic("node number is not matched")
+	}
+}
@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"hash"
+	"io"
 	"math/big"
 	"math/rand"
 	"reflect"
@@ -362,7 +363,9 @@ func TestRandomCases(t *testing.T) {
 		{op: 1, key: common.Hex2Bytes("980c393656413a15c8da01978ed9f89feb80b502f58f2d640e3a2f5f7a99a7018f1b573befd92053ac6f78fca4a87268"), value: common.Hex2Bytes("")}, // step 24
 		{op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")},                                                                                               // step 25
 	}
-	runRandTest(rt)
+	if err := runRandTest(rt); err != nil {
+		t.Fatal(err)
+	}
 }

 // randTest performs random trie operations.
@@ -389,33 +392,45 @@ const (
 )

 func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
+	var finishedFn = func() bool {
+		size--
+		return size > 0
+	}
+	return reflect.ValueOf(generateSteps(finishedFn, r))
+}
+
+func generateSteps(finished func() bool, r io.Reader) randTest {
 	var allKeys [][]byte
+	var one = []byte{0}
 	genKey := func() []byte {
-		if len(allKeys) < 2 || r.Intn(100) < 10 {
+		r.Read(one)
+		if len(allKeys) < 2 || one[0]%100 > 90 {
 			// new key
-			key := make([]byte, r.Intn(50))
+			size := one[0] % 50
+			key := make([]byte, size)
 			r.Read(key)
 			allKeys = append(allKeys, key)
 			return key
 		}
 		// use existing key
-		return allKeys[r.Intn(len(allKeys))]
+		idx := int(one[0]) % len(allKeys)
+		return allKeys[idx]
 	}
-
 	var steps randTest
-	for i := 0; i < size; i++ {
-		step := randTestStep{op: r.Intn(opMax)}
+	for !finished() {
+		r.Read(one)
+		step := randTestStep{op: int(one[0]) % opMax}
 		switch step.op {
 		case opUpdate:
 			step.key = genKey()
 			step.value = make([]byte, 8)
-			binary.BigEndian.PutUint64(step.value, uint64(i))
+			binary.BigEndian.PutUint64(step.value, uint64(len(steps)))
 		case opGet, opDelete, opProve:
 			step.key = genKey()
 		}
 		steps = append(steps, step)
 	}
-	return reflect.ValueOf(steps)
+	return steps
 }

 func verifyAccessList(old *Trie, new *Trie, set *trienode.NodeSet) error {
@@ -460,7 +475,12 @@ func verifyAccessList(old *Trie, new *Trie, set *trienode.NodeSet) error {
 	return nil
 }

-func runRandTest(rt randTest) bool {
+// runRandTestBool coerces error to boolean, for use in quick.Check
+func runRandTestBool(rt randTest) bool {
+	return runRandTest(rt) == nil
+}
+
+func runRandTest(rt randTest) error {
 	var scheme = rawdb.HashScheme
 	if rand.Intn(2) == 0 {
 		scheme = rawdb.PathScheme
@@ -513,12 +533,12 @@ func runRandTest(rt randTest) bool {
 			newtr, err := New(TrieID(root), triedb)
 			if err != nil {
 				rt[i].err = err
-				return false
+				return err
 			}
 			if nodes != nil {
 				if err := verifyAccessList(origTrie, newtr, nodes); err != nil {
 					rt[i].err = err
-					return false
+					return err
 				}
 			}
 			tr = newtr
@@ -587,14 +607,14 @@ func runRandTest(rt randTest) bool {
 		}
 		// Abort the test on error.
 		if rt[i].err != nil {
-			return false
+			return rt[i].err
 		}
 	}
-	return true
+	return nil
 }

 func TestRandom(t *testing.T) {
-	if err := quick.Check(runRandTest, nil); err != nil {
+	if err := quick.Check(runRandTestBool, nil); err != nil {
 		if cerr, ok := err.(*quick.CheckError); ok {
 			t.Fatalf("random test iteration %d failed: %s", cerr.Count, spew.Sdump(cerr.In))
 		}
@@ -1185,3 +1205,17 @@ func TestDecodeNode(t *testing.T) {
 		decodeNode(hash, elems)
 	}
 }
+
+func FuzzTrie(f *testing.F) {
+	f.Fuzz(func(t *testing.T, data []byte) {
+		var steps = 500
+		var input = bytes.NewReader(data)
+		var finishedFn = func() bool {
+			steps--
+			return steps < 0 || input.Len() == 0
+		}
+		if err := runRandTest(generateSteps(finishedFn, input)); err != nil {
+			t.Fatal(err)
+		}
+	})
+}