From efe6dd29042b36d543420a422fc21d123f1e67e3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 11 Dec 2020 11:06:44 +0100 Subject: [PATCH] consensus/ethash: implement faster difficulty calculators (#21976) This PR adds re-written difficulty calculators, which are based on uint256. It also adds a fuzzer + oss-fuzz integration for the new fuzzer. It does differential fuzzing between the new and old calculators. Note: this PR does not actually enable the new calculators. --- consensus/ethash/consensus.go | 5 + consensus/ethash/consensus_test.go | 102 +++++++++++ consensus/ethash/difficulty.go | 193 ++++++++++++++++++++ oss-fuzz.sh | 1 + tests/fuzzers/difficulty/debug/main.go | 23 +++ tests/fuzzers/difficulty/difficulty-fuzz.go | 145 +++++++++++++++ 6 files changed, 469 insertions(+) create mode 100644 consensus/ethash/difficulty.go create mode 100644 tests/fuzzers/difficulty/debug/main.go create mode 100644 tests/fuzzers/difficulty/difficulty-fuzz.go diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index bdc02098a..8e401af7c 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -485,6 +485,11 @@ func calcDifficultyFrontier(time uint64, parent *types.Header) *big.Int { return diff } +// Exported for fuzzing +var FrontierDifficultyCalulator = calcDifficultyFrontier +var HomesteadDifficultyCalulator = calcDifficultyHomestead +var DynamicDifficultyCalculator = makeDifficultyCalculator + // VerifySeal implements consensus.Engine, checking whether the given block satisfies // the PoW difficulty requirements. func (ethash *Ethash) VerifySeal(chain consensus.ChainHeaderReader, header *types.Header) error { diff --git a/consensus/ethash/consensus_test.go b/consensus/ethash/consensus_test.go index 675737d9e..6f6dc79fd 100644 --- a/consensus/ethash/consensus_test.go +++ b/consensus/ethash/consensus_test.go @@ -17,12 +17,15 @@ package ethash import ( + "encoding/binary" "encoding/json" "math/big" + "math/rand" "os" "path/filepath" "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/params" @@ -84,3 +87,102 @@ func TestCalcDifficulty(t *testing.T) { } } } + +func randSlice(min, max uint32) []byte { + var b = make([]byte, 4) + rand.Read(b) + a := binary.LittleEndian.Uint32(b) + size := min + a%(max-min) + out := make([]byte, size) + rand.Read(out) + return out +} + +func TestDifficultyCalculators(t *testing.T) { + rand.Seed(2) + for i := 0; i < 5000; i++ { + // 1 to 300 seconds diff + var timeDelta = uint64(1 + rand.Uint32()%3000) + diffBig := big.NewInt(0).SetBytes(randSlice(2, 10)) + if diffBig.Cmp(params.MinimumDifficulty) < 0 { + diffBig.Set(params.MinimumDifficulty) + } + //rand.Read(difficulty) + header := &types.Header{ + Difficulty: diffBig, + Number: new(big.Int).SetUint64(rand.Uint64() % 50_000_000), + Time: rand.Uint64() - timeDelta, + } + if rand.Uint32()&1 == 0 { + header.UncleHash = types.EmptyUncleHash + } + bombDelay := new(big.Int).SetUint64(rand.Uint64() % 50_000_000) + for i, pair := range []struct { + bigFn func(time uint64, parent *types.Header) *big.Int + u256Fn func(time uint64, parent *types.Header) *big.Int + }{ + {FrontierDifficultyCalulator, CalcDifficultyFrontierU256}, + {HomesteadDifficultyCalulator, CalcDifficultyHomesteadU256}, + {DynamicDifficultyCalculator(bombDelay), MakeDifficultyCalculatorU256(bombDelay)}, + } { + time := header.Time + timeDelta + want := pair.bigFn(time, header) + have := pair.u256Fn(time, header) + if want.BitLen() > 256 { + continue + } + if want.Cmp(have) != 0 { + t.Fatalf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have, + header.Number, header.Time, time, bombDelay) + } + } + } +} + +func BenchmarkDifficultyCalculator(b *testing.B) { + x1 := makeDifficultyCalculator(big.NewInt(1000000)) + x2 := MakeDifficultyCalculatorU256(big.NewInt(1000000)) + h := &types.Header{ + ParentHash: common.Hash{}, + UncleHash: types.EmptyUncleHash, + Difficulty: big.NewInt(0xffffff), + Number: big.NewInt(500000), + Time: 1000000, + } + b.Run("big-frontier", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + calcDifficultyFrontier(1000014, h) + } + }) + b.Run("u256-frontier", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + CalcDifficultyFrontierU256(1000014, h) + } + }) + b.Run("big-homestead", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + calcDifficultyHomestead(1000014, h) + } + }) + b.Run("u256-homestead", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + CalcDifficultyHomesteadU256(1000014, h) + } + }) + b.Run("big-generic", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + x1(1000014, h) + } + }) + b.Run("u256-generic", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + x2(1000014, h) + } + }) +} diff --git a/consensus/ethash/difficulty.go b/consensus/ethash/difficulty.go new file mode 100644 index 000000000..59c4ac741 --- /dev/null +++ b/consensus/ethash/difficulty.go @@ -0,0 +1,193 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethash + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/holiman/uint256" +) + +const ( + // frontierDurationLimit is for Frontier: + // The decision boundary on the blocktime duration used to determine + // whether difficulty should go up or down. + frontierDurationLimit = 13 + // minimumDifficulty The minimum that the difficulty may ever be. + minimumDifficulty = 131072 + // expDiffPeriod is the exponential difficulty period + expDiffPeriodUint = 100000 + // difficultyBoundDivisorBitShift is the bound divisor of the difficulty (2048), + // This constant is the right-shifts to use for the division. + difficultyBoundDivisor = 11 +) + +// CalcDifficultyFrontierU256 is the difficulty adjustment algorithm. It returns the +// difficulty that a new block should have when created at time given the parent +// block's time and difficulty. The calculation uses the Frontier rules. +func CalcDifficultyFrontierU256(time uint64, parent *types.Header) *big.Int { + /* + Algorithm + block_diff = pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1) + int(2^((num // 100000) - 2)) + + Where: + - pdiff = parent.difficulty + - ptime = parent.time + - time = block.timestamp + - num = block.number + */ + + pDiff := uint256.NewInt() + pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + adjust := pDiff.Clone() + adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 + + if time-parent.Time < frontierDurationLimit { + pDiff.Add(pDiff, adjust) + } else { + pDiff.Sub(pDiff, adjust) + } + if pDiff.LtUint64(minimumDifficulty) { + pDiff.SetUint64(minimumDifficulty) + } + // 'pdiff' now contains: + // pdiff + pdiff / 2048 * (1 if time - ptime < 13 else -1) + + if periodCount := (parent.Number.Uint64() + 1) / expDiffPeriodUint; periodCount > 1 { + // diff = diff + 2^(periodCount - 2) + expDiff := adjust.SetOne() + expDiff.Lsh(expDiff, uint(periodCount-2)) // expdiff: 2 ^ (periodCount -2) + pDiff.Add(pDiff, expDiff) + } + return pDiff.ToBig() +} + +// CalcDifficultyHomesteadU256 is the difficulty adjustment algorithm. It returns +// the difficulty that a new block should have when created at time given the +// parent block's time and difficulty. The calculation uses the Homestead rules. +func CalcDifficultyHomesteadU256(time uint64, parent *types.Header) *big.Int { + /* + https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2.md + Algorithm: + block_diff = pdiff + pdiff / 2048 * max(1 - (time - ptime) / 10, -99) + 2 ^ int((num / 100000) - 2)) + + Our modification, to use unsigned ints: + block_diff = pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + 2 ^ int((num / 100000) - 2)) + + Where: + - pdiff = parent.difficulty + - ptime = parent.time + - time = block.timestamp + - num = block.number + */ + + pDiff := uint256.NewInt() + pDiff.SetFromBig(parent.Difficulty) // pDiff: pdiff + adjust := pDiff.Clone() + adjust.Rsh(adjust, difficultyBoundDivisor) // adjust: pDiff / 2048 + + x := (time - parent.Time) / 10 // (time - ptime) / 10) + var neg = true + if x == 0 { + x = 1 + neg = false + } else if x >= 100 { + x = 99 + } else { + x = x - 1 + } + z := new(uint256.Int).SetUint64(x) + adjust.Mul(adjust, z) // adjust: (pdiff / 2048) * max((time - ptime) / 10 - 1, 99) + if neg { + pDiff.Sub(pDiff, adjust) // pdiff - pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + } else { + pDiff.Add(pDiff, adjust) // pdiff + pdiff / 2048 * max((time - ptime) / 10 - 1, 99) + } + if pDiff.LtUint64(minimumDifficulty) { + pDiff.SetUint64(minimumDifficulty) + } + // for the exponential factor, a.k.a "the bomb" + // diff = diff + 2^(periodCount - 2) + if periodCount := (1 + parent.Number.Uint64()) / expDiffPeriodUint; periodCount > 1 { + expFactor := adjust.Lsh(adjust.SetOne(), uint(periodCount-2)) + pDiff.Add(pDiff, expFactor) + } + return pDiff.ToBig() +} + +// MakeDifficultyCalculatorU256 creates a difficultyCalculator with the given bomb-delay. +// the difficulty is calculated with Byzantium rules, which differs from Homestead in +// how uncles affect the calculation +func MakeDifficultyCalculatorU256(bombDelay *big.Int) func(time uint64, parent *types.Header) *big.Int { + // Note, the calculations below looks at the parent number, which is 1 below + // the block number. Thus we remove one from the delay given + bombDelayFromParent := bombDelay.Uint64() - 1 + return func(time uint64, parent *types.Header) *big.Int { + /* + https://github.com/ethereum/EIPs/issues/100 + pDiff = parent.difficulty + BLOCK_DIFF_FACTOR = 9 + a = pDiff + (pDiff // BLOCK_DIFF_FACTOR) * adj_factor + b = min(parent.difficulty, MIN_DIFF) + child_diff = max(a,b ) + */ + x := (time - parent.Time) / 9 // (block_timestamp - parent_timestamp) // 9 + c := uint64(1) // if parent.unclehash == emptyUncleHashHash + if parent.UncleHash != types.EmptyUncleHash { + c = 2 + } + xNeg := x >= c + if xNeg { + // x is now _negative_ adjustment factor + x = x - c // - ( (t-p)/p -( 2 or 1) ) + } else { + x = c - x // (2 or 1) - (t-p)/9 + } + if x > 99 { + x = 99 // max(x, 99) + } + // parent_diff + (parent_diff / 2048 * max((2 if len(parent.uncles) else 1) - ((timestamp - parent.timestamp) // 9), -99)) + y := new(uint256.Int) + y.SetFromBig(parent.Difficulty) // y: p_diff + pDiff := y.Clone() // pdiff: p_diff + z := new(uint256.Int).SetUint64(x) //z : +-adj_factor (either pos or negative) + y.Rsh(y, difficultyBoundDivisor) // y: p__diff / 2048 + z.Mul(y, z) // z: (p_diff / 2048 ) * (+- adj_factor) + + if xNeg { + y.Sub(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor + } else { + y.Add(pDiff, z) // y: parent_diff + parent_diff/2048 * adjustment_factor + } + // minimum difficulty can ever be (before exponential factor) + if y.LtUint64(minimumDifficulty) { + y.SetUint64(minimumDifficulty) + } + // calculate a fake block number for the ice-age delay + // Specification: https://eips.ethereum.org/EIPS/eip-1234 + var pNum = parent.Number.Uint64() + if pNum >= bombDelayFromParent { + if fakeBlockNumber := pNum - bombDelayFromParent; fakeBlockNumber >= 2*expDiffPeriodUint { + z.SetOne() + z.Lsh(z, uint(fakeBlockNumber/expDiffPeriodUint-2)) + y.Add(z, y) + } + } + return y.ToBig() + } +} diff --git a/oss-fuzz.sh b/oss-fuzz.sh index e0a293a6d..e060ea88e 100644 --- a/oss-fuzz.sh +++ b/oss-fuzz.sh @@ -57,6 +57,7 @@ compile_fuzzer tests/fuzzers/txfetcher Fuzz fuzzTxfetcher compile_fuzzer tests/fuzzers/rlp Fuzz fuzzRlp compile_fuzzer tests/fuzzers/trie Fuzz fuzzTrie compile_fuzzer tests/fuzzers/stacktrie Fuzz fuzzStackTrie +compile_fuzzer tests/fuzzers/difficulty Fuzz fuzzDifficulty compile_fuzzer tests/fuzzers/bls12381 FuzzG1Add fuzz_g1_add compile_fuzzer tests/fuzzers/bls12381 FuzzG1Mul fuzz_g1_mul diff --git a/tests/fuzzers/difficulty/debug/main.go b/tests/fuzzers/difficulty/debug/main.go new file mode 100644 index 000000000..23516b3a0 --- /dev/null +++ b/tests/fuzzers/difficulty/debug/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + + "github.com/ethereum/go-ethereum/tests/fuzzers/difficulty" +) + +func main() { + if len(os.Args) != 2 { + fmt.Fprintf(os.Stderr, "Usage: debug ") + os.Exit(1) + } + crasher := os.Args[1] + data, err := ioutil.ReadFile(crasher) + if err != nil { + fmt.Fprintf(os.Stderr, "error loading crasher %v: %v", crasher, err) + os.Exit(1) + } + difficulty.Fuzz(data) +} diff --git a/tests/fuzzers/difficulty/difficulty-fuzz.go b/tests/fuzzers/difficulty/difficulty-fuzz.go new file mode 100644 index 000000000..e4c5dcf57 --- /dev/null +++ b/tests/fuzzers/difficulty/difficulty-fuzz.go @@ -0,0 +1,145 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package difficulty + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core/types" +) + +type fuzzer struct { + input io.Reader + exhausted bool + debugging bool +} + +func (f *fuzzer) read(size int) []byte { + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) readSlice(min, max int) []byte { + var a uint16 + binary.Read(f.input, binary.LittleEndian, &a) + size := min + int(a)%(max-min) + out := make([]byte, size) + if _, err := f.input.Read(out); err != nil { + f.exhausted = true + } + return out +} + +func (f *fuzzer) readUint64(min, max uint64) uint64 { + if min == max { + return min + } + var a uint64 + if err := binary.Read(f.input, binary.LittleEndian, &a); err != nil { + f.exhausted = true + } + a = min + a%(max-min) + return a +} +func (f *fuzzer) readBool() bool { + return f.read(1)[0]&0x1 == 0 +} + +// The function must return +// 1 if the fuzzer should increase priority of the +// given input during subsequent fuzzing (for example, the input is lexically +// correct and was parsed successfully); +// -1 if the input must not be added to corpus even if gives new coverage; and +// 0 otherwise +// other values are reserved for future use. +func Fuzz(data []byte) int { + f := fuzzer{ + input: bytes.NewReader(data), + exhausted: false, + } + return f.fuzz() +} + +var minDifficulty = big.NewInt(0x2000) + +type calculator func(time uint64, parent *types.Header) *big.Int + +func (f *fuzzer) fuzz() int { + // A parent header + header := &types.Header{} + if f.readBool() { + header.UncleHash = types.EmptyUncleHash + } + // Difficulty can range between 0x2000 (2 bytes) and up to 32 bytes + { + diff := new(big.Int).SetBytes(f.readSlice(2, 32)) + if diff.Cmp(minDifficulty) < 0 { + diff.Set(minDifficulty) + } + header.Difficulty = diff + } + // Number can range between 0 and up to 32 bytes (but not so that the child exceeds it) + { + // However, if we use astronomic numbers, then the bomb exp karatsuba calculation + // in the legacy methods) + // times out, so we limit it to fit within reasonable bounds + number := new(big.Int).SetBytes(f.readSlice(0, 4)) // 4 bytes: 32 bits: block num max 4 billion + header.Number = number + } + // Both parent and child time must fit within uint64 + var time uint64 + { + childTime := f.readUint64(1, 0xFFFFFFFFFFFFFFFF) + //fmt.Printf("childTime: %x\n",childTime) + delta := f.readUint64(1, childTime) + //fmt.Printf("delta: %v\n", delta) + pTime := childTime - delta + header.Time = pTime + time = childTime + } + // Bomb delay will never exceed uint64 + bombDelay := new(big.Int).SetUint64(f.readUint64(1, 0xFFFFFFFFFFFFFFFe)) + + if f.exhausted { + return 0 + } + + for i, pair := range []struct { + bigFn calculator + u256Fn calculator + }{ + {ethash.FrontierDifficultyCalulator, ethash.CalcDifficultyFrontierU256}, + {ethash.HomesteadDifficultyCalulator, ethash.CalcDifficultyHomesteadU256}, + {ethash.DynamicDifficultyCalculator(bombDelay), ethash.MakeDifficultyCalculatorU256(bombDelay)}, + } { + want := pair.bigFn(time, header) + have := pair.u256Fn(time, header) + if want.Cmp(have) != 0 { + panic(fmt.Sprintf("pair %d: want %x have %x\nparent.Number: %x\np.Time: %x\nc.Time: %x\nBombdelay: %v\n", i, want, have, + header.Number, header.Time, time, bombDelay)) + } + } + return 1 +}