diff --git a/cmd/geth/main.go b/cmd/geth/main.go
index 3422d9500..3e3a3eb03 100644
--- a/cmd/geth/main.go
+++ b/cmd/geth/main.go
@@ -304,7 +304,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
utils.DataDirFlag,
utils.BlockchainVersionFlag,
utils.OlympicFlag,
- utils.EthVersionFlag,
+ utils.FastSyncFlag,
utils.CacheFlag,
utils.JSpathFlag,
utils.ListenPortFlag,
@@ -360,7 +360,6 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
utils.SetupLogger(ctx)
utils.SetupNetwork(ctx)
utils.SetupVM(ctx)
- utils.SetupEth(ctx)
if ctx.GlobalBool(utils.PProfEanbledFlag.Name) {
utils.StartPProf(ctx)
}
diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go
index ca9dd76fd..060d80b35 100644
--- a/cmd/utils/flags.go
+++ b/cmd/utils/flags.go
@@ -148,10 +148,9 @@ var (
Name: "olympic",
Usage: "Use olympic style protocol",
}
- EthVersionFlag = cli.IntFlag{
- Name: "eth",
- Value: 62,
- Usage: "Highest eth protocol to advertise (temporary, dev option)",
+ FastSyncFlag = cli.BoolFlag{
+ Name: "fast",
+ Usage: "Enables fast syncing through state downloads",
}
// miner settings
@@ -425,12 +424,13 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config {
if err != nil {
glog.V(logger.Error).Infoln("WARNING: No etherbase set and no accounts found as default")
}
-
+ // Assemble the entire eth configuration and return
cfg := ð.Config{
Name: common.MakeName(clientID, version),
DataDir: MustDataDir(ctx),
GenesisNonce: ctx.GlobalInt(GenesisNonceFlag.Name),
GenesisFile: ctx.GlobalString(GenesisFileFlag.Name),
+ FastSync: ctx.GlobalBool(FastSyncFlag.Name),
BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name),
DatabaseCache: ctx.GlobalInt(CacheFlag.Name),
SkipBcVersionCheck: false,
@@ -499,7 +499,6 @@ func MakeEthConfig(clientID, version string, ctx *cli.Context) *eth.Config {
glog.V(logger.Info).Infoln("dev mode enabled")
}
-
return cfg
}
@@ -532,18 +531,6 @@ func SetupVM(ctx *cli.Context) {
vm.SetJITCacheSize(ctx.GlobalInt(VMJitCacheFlag.Name))
}
-// SetupEth configures the eth packages global settings
-func SetupEth(ctx *cli.Context) {
- version := ctx.GlobalInt(EthVersionFlag.Name)
- for len(eth.ProtocolVersions) > 0 && eth.ProtocolVersions[0] > uint(version) {
- eth.ProtocolVersions = eth.ProtocolVersions[1:]
- eth.ProtocolLengths = eth.ProtocolLengths[1:]
- }
- if len(eth.ProtocolVersions) == 0 {
- Fatalf("No valid eth protocols remaining")
- }
-}
-
// MakeChain creates a chain manager from set command line flags.
func MakeChain(ctx *cli.Context) (chain *core.BlockChain, chainDb ethdb.Database) {
datadir := MustDataDir(ctx)
diff --git a/core/bench_test.go b/core/bench_test.go
index 27f3e3158..b5eb51803 100644
--- a/core/bench_test.go
+++ b/core/bench_test.go
@@ -163,7 +163,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) {
// Generate a chain of b.N blocks using the supplied block
// generator function.
genesis := WriteGenesisBlockForTesting(db, GenesisAccount{benchRootAddr, benchRootFunds})
- chain := GenerateChain(genesis, db, b.N, gen)
+ chain, _ := GenerateChain(genesis, db, b.N, gen)
// Time the insertion of the new chain.
// State and blocks are stored in the same DB.
diff --git a/core/block_processor.go b/core/block_processor.go
index 7032c077c..e7b2f63e5 100644
--- a/core/block_processor.go
+++ b/core/block_processor.go
@@ -128,7 +128,7 @@ func (self *BlockProcessor) ApplyTransaction(gp *GasPool, statedb *state.StateDB
}
logs := statedb.GetLogs(tx.Hash())
- receipt.SetLogs(logs)
+ receipt.Logs = logs
receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
glog.V(logger.Debug).Infoln(receipt)
@@ -212,14 +212,16 @@ func (sm *BlockProcessor) Process(block *types.Block) (logs vm.Logs, receipts ty
defer sm.mutex.Unlock()
if sm.bc.HasBlock(block.Hash()) {
- return nil, nil, &KnownBlockError{block.Number(), block.Hash()}
+ if _, err := state.New(block.Root(), sm.chainDb); err == nil {
+ return nil, nil, &KnownBlockError{block.Number(), block.Hash()}
+ }
}
-
- if !sm.bc.HasBlock(block.ParentHash()) {
- return nil, nil, ParentError(block.ParentHash())
+ if parent := sm.bc.GetBlock(block.ParentHash()); parent != nil {
+ if _, err := state.New(parent.Root(), sm.chainDb); err == nil {
+ return sm.processWithParent(block, parent)
+ }
}
- parent := sm.bc.GetBlock(block.ParentHash())
- return sm.processWithParent(block, parent)
+ return nil, nil, ParentError(block.ParentHash())
}
func (sm *BlockProcessor) processWithParent(block, parent *types.Block) (logs vm.Logs, receipts types.Receipts, err error) {
@@ -381,18 +383,40 @@ func (sm *BlockProcessor) GetLogs(block *types.Block) (logs vm.Logs, err error)
receipts := GetBlockReceipts(sm.chainDb, block.Hash())
// coalesce logs
for _, receipt := range receipts {
- logs = append(logs, receipt.Logs()...)
+ logs = append(logs, receipt.Logs...)
}
return logs, nil
}
+// ValidateHeader verifies the validity of a header, relying on the database and
+// POW behind the block processor.
+func (sm *BlockProcessor) ValidateHeader(header *types.Header, checkPow, uncle bool) error {
+ // Short circuit if the header's already known or its parent missing
+ if sm.bc.HasHeader(header.Hash()) {
+ return nil
+ }
+ if parent := sm.bc.GetHeader(header.ParentHash); parent == nil {
+ return ParentError(header.ParentHash)
+ } else {
+ return ValidateHeader(sm.Pow, header, parent, checkPow, uncle)
+ }
+}
+
+// ValidateHeaderWithParent verifies the validity of a header, relying on the database and
+// POW behind the block processor.
+func (sm *BlockProcessor) ValidateHeaderWithParent(header, parent *types.Header, checkPow, uncle bool) error {
+ if sm.bc.HasHeader(header.Hash()) {
+ return nil
+ }
+ return ValidateHeader(sm.Pow, header, parent, checkPow, uncle)
+}
+
// See YP section 4.3.4. "Block Header Validity"
// Validates a header. Returns an error if the header is invalid.
func ValidateHeader(pow pow.PoW, header *types.Header, parent *types.Header, checkPow, uncle bool) error {
if big.NewInt(int64(len(header.Extra))).Cmp(params.MaximumExtraDataSize) == 1 {
return fmt.Errorf("Header extra data too long (%d)", len(header.Extra))
}
-
if uncle {
if header.Time.Cmp(common.MaxBig) == 1 {
return BlockTSTooBigErr
@@ -429,7 +453,7 @@ func ValidateHeader(pow pow.PoW, header *types.Header, parent *types.Header, che
if checkPow {
// Verify the nonce of the header. Return an error if it's not valid
if !pow.Verify(types.NewBlockWithHeader(header)) {
- return ValidationError("Header's nonce is invalid (= %x)", header.Nonce)
+ return &BlockNonceErr{Hash: header.Hash(), Number: header.Number, Nonce: header.Nonce.Uint64()}
}
}
return nil
diff --git a/core/block_processor_test.go b/core/block_processor_test.go
index e0e5607b9..3050456b4 100644
--- a/core/block_processor_test.go
+++ b/core/block_processor_test.go
@@ -70,16 +70,16 @@ func TestPutReceipt(t *testing.T) {
hash[0] = 2
receipt := new(types.Receipt)
- receipt.SetLogs(vm.Logs{&vm.Log{
- Address: addr,
- Topics: []common.Hash{hash},
- Data: []byte("hi"),
- Number: 42,
- TxHash: hash,
- TxIndex: 0,
- BlockHash: hash,
- Index: 0,
- }})
+ receipt.Logs = vm.Logs{&vm.Log{
+ Address: addr,
+ Topics: []common.Hash{hash},
+ Data: []byte("hi"),
+ BlockNumber: 42,
+ TxHash: hash,
+ TxIndex: 0,
+ BlockHash: hash,
+ Index: 0,
+ }}
PutReceipts(db, types.Receipts{receipt})
receipt = GetReceipt(db, common.Hash{})
diff --git a/core/blockchain.go b/core/blockchain.go
index 7bfe13a11..f14ff363c 100644
--- a/core/blockchain.go
+++ b/core/blockchain.go
@@ -18,10 +18,14 @@
package core
import (
+ crand "crypto/rand"
"errors"
"fmt"
"io"
+ "math"
"math/big"
+ mrand "math/rand"
+ "runtime"
"sync"
"sync/atomic"
"time"
@@ -29,6 +33,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/logger"
@@ -36,6 +41,7 @@ import (
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/pow"
"github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
"github.com/hashicorp/golang-lru"
)
@@ -67,9 +73,10 @@ type BlockChain struct {
chainmu sync.RWMutex
tsmu sync.RWMutex
- td *big.Int
- currentBlock *types.Block
- currentGasLimit *big.Int
+ checkpoint int // checkpoint counts towards the new checkpoint
+ currentHeader *types.Header // Current head of the header chain (may be above the block chain!)
+ currentBlock *types.Block // Current head of the block chain
+ currentFastBlock *types.Block // Current head of the fast-sync chain (may be above the block chain!)
headerCache *lru.Cache // Cache for the most recent block headers
bodyCache *lru.Cache // Cache for the most recent block bodies
@@ -84,7 +91,8 @@ type BlockChain struct {
procInterrupt int32 // interrupt signaler for block processing
wg sync.WaitGroup
- pow pow.PoW
+ pow pow.PoW
+ rand *mrand.Rand
}
func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*BlockChain, error) {
@@ -107,6 +115,12 @@ func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*Bl
futureBlocks: futureBlocks,
pow: pow,
}
+ // Seed a fast but crypto originating random generator
+ seed, err := crand.Int(crand.Reader, big.NewInt(math.MaxInt64))
+ if err != nil {
+ return nil, err
+ }
+ bc.rand = mrand.New(mrand.NewSource(seed.Int64()))
bc.genesisBlock = bc.GetBlockByNumber(0)
if bc.genesisBlock == nil {
@@ -120,20 +134,15 @@ func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*Bl
}
glog.V(logger.Info).Infoln("WARNING: Wrote default ethereum genesis block")
}
- if err := bc.setLastState(); err != nil {
+ if err := bc.loadLastState(); err != nil {
return nil, err
}
// Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain
for hash, _ := range BadHashes {
- if block := bc.GetBlock(hash); block != nil {
- glog.V(logger.Error).Infof("Found bad hash. Reorganising chain to state %x\n", block.ParentHash().Bytes()[:4])
- block = bc.GetBlock(block.ParentHash())
- if block == nil {
- glog.Fatal("Unable to complete. Parent block not found. Corrupted DB?")
- }
- bc.SetHead(block)
-
- glog.V(logger.Error).Infoln("Chain reorg was successfull. Resuming normal operation")
+ if header := bc.GetHeader(hash); header != nil {
+ glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4])
+ bc.SetHead(header.Number.Uint64() - 1)
+ glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation")
}
}
// Take ownership of this particular state
@@ -141,30 +150,146 @@ func NewBlockChain(chainDb ethdb.Database, pow pow.PoW, mux *event.TypeMux) (*Bl
return bc, nil
}
-func (bc *BlockChain) SetHead(head *types.Block) {
+// loadLastState loads the last known chain state from the database. This method
+// assumes that the chain manager mutex is held.
+func (self *BlockChain) loadLastState() error {
+ // Restore the last known head block
+ head := GetHeadBlockHash(self.chainDb)
+ if head == (common.Hash{}) {
+ // Corrupt or empty database, init from scratch
+ self.Reset()
+ } else {
+ if block := self.GetBlock(head); block != nil {
+ // Block found, set as the current head
+ self.currentBlock = block
+ } else {
+ // Corrupt or empty database, init from scratch
+ self.Reset()
+ }
+ }
+ // Restore the last known head header
+ self.currentHeader = self.currentBlock.Header()
+ if head := GetHeadHeaderHash(self.chainDb); head != (common.Hash{}) {
+ if header := self.GetHeader(head); header != nil {
+ self.currentHeader = header
+ }
+ }
+ // Restore the last known head fast block
+ self.currentFastBlock = self.currentBlock
+ if head := GetHeadFastBlockHash(self.chainDb); head != (common.Hash{}) {
+ if block := self.GetBlock(head); block != nil {
+ self.currentFastBlock = block
+ }
+ }
+ // Issue a status log and return
+ headerTd := self.GetTd(self.currentHeader.Hash())
+ blockTd := self.GetTd(self.currentBlock.Hash())
+ fastTd := self.GetTd(self.currentFastBlock.Hash())
+
+ glog.V(logger.Info).Infof("Last header: #%d [%x…] TD=%v", self.currentHeader.Number, self.currentHeader.Hash().Bytes()[:4], headerTd)
+ glog.V(logger.Info).Infof("Last block: #%d [%x…] TD=%v", self.currentBlock.Number(), self.currentBlock.Hash().Bytes()[:4], blockTd)
+ glog.V(logger.Info).Infof("Fast block: #%d [%x…] TD=%v", self.currentFastBlock.Number(), self.currentFastBlock.Hash().Bytes()[:4], fastTd)
+
+ return nil
+}
+
+// SetHead rewinds the local chain to a new head. In the case of headers, everything
+// above the new head will be deleted and the new one set. In the case of blocks
+// though, the head may be further rewound if block bodies are missing (non-archive
+// nodes after a fast sync).
+func (bc *BlockChain) SetHead(head uint64) {
bc.mu.Lock()
defer bc.mu.Unlock()
- for block := bc.currentBlock; block != nil && block.Hash() != head.Hash(); block = bc.GetBlock(block.ParentHash()) {
- DeleteBlock(bc.chainDb, block.Hash())
+ // Figure out the highest known canonical headers and/or blocks
+ height := uint64(0)
+ if bc.currentHeader != nil {
+ if hh := bc.currentHeader.Number.Uint64(); hh > height {
+ height = hh
+ }
}
+ if bc.currentBlock != nil {
+ if bh := bc.currentBlock.NumberU64(); bh > height {
+ height = bh
+ }
+ }
+ if bc.currentFastBlock != nil {
+ if fbh := bc.currentFastBlock.NumberU64(); fbh > height {
+ height = fbh
+ }
+ }
+ // Gather all the hashes that need deletion
+ drop := make(map[common.Hash]struct{})
+
+ for bc.currentHeader != nil && bc.currentHeader.Number.Uint64() > head {
+ drop[bc.currentHeader.Hash()] = struct{}{}
+ bc.currentHeader = bc.GetHeader(bc.currentHeader.ParentHash)
+ }
+ for bc.currentBlock != nil && bc.currentBlock.NumberU64() > head {
+ drop[bc.currentBlock.Hash()] = struct{}{}
+ bc.currentBlock = bc.GetBlock(bc.currentBlock.ParentHash())
+ }
+ for bc.currentFastBlock != nil && bc.currentFastBlock.NumberU64() > head {
+ drop[bc.currentFastBlock.Hash()] = struct{}{}
+ bc.currentFastBlock = bc.GetBlock(bc.currentFastBlock.ParentHash())
+ }
+ // Roll back the canonical chain numbering
+ for i := height; i > head; i-- {
+ DeleteCanonicalHash(bc.chainDb, i)
+ }
+ // Delete everything found by the above rewind
+ for hash, _ := range drop {
+ DeleteHeader(bc.chainDb, hash)
+ DeleteBody(bc.chainDb, hash)
+ DeleteTd(bc.chainDb, hash)
+ }
+ // Clear out any stale content from the caches
bc.headerCache.Purge()
bc.bodyCache.Purge()
bc.bodyRLPCache.Purge()
bc.blockCache.Purge()
bc.futureBlocks.Purge()
- bc.currentBlock = head
- bc.setTotalDifficulty(bc.GetTd(head.Hash()))
- bc.insert(head)
- bc.setLastState()
+ // Update all computed fields to the new head
+ if bc.currentBlock == nil {
+ bc.currentBlock = bc.genesisBlock
+ }
+ if bc.currentHeader == nil {
+ bc.currentHeader = bc.genesisBlock.Header()
+ }
+ if bc.currentFastBlock == nil {
+ bc.currentFastBlock = bc.genesisBlock
+ }
+ if err := WriteHeadBlockHash(bc.chainDb, bc.currentBlock.Hash()); err != nil {
+ glog.Fatalf("failed to reset head block hash: %v", err)
+ }
+ if err := WriteHeadHeaderHash(bc.chainDb, bc.currentHeader.Hash()); err != nil {
+ glog.Fatalf("failed to reset head header hash: %v", err)
+ }
+ if err := WriteHeadFastBlockHash(bc.chainDb, bc.currentFastBlock.Hash()); err != nil {
+ glog.Fatalf("failed to reset head fast block hash: %v", err)
+ }
+ bc.loadLastState()
}
-func (self *BlockChain) Td() *big.Int {
- self.mu.RLock()
- defer self.mu.RUnlock()
+// FastSyncCommitHead sets the current head block to the one defined by the hash
+// irrelevant what the chain contents were prior.
+func (self *BlockChain) FastSyncCommitHead(hash common.Hash) error {
+ // Make sure that both the block as well at its state trie exists
+ block := self.GetBlock(hash)
+ if block == nil {
+ return fmt.Errorf("non existent block [%x…]", hash[:4])
+ }
+ if _, err := trie.NewSecure(block.Root(), self.chainDb); err != nil {
+ return err
+ }
+ // If all checks out, manually set the head block
+ self.mu.Lock()
+ self.currentBlock = block
+ self.mu.Unlock()
- return new(big.Int).Set(self.td)
+ glog.V(logger.Info).Infof("committed block #%d [%x…] as new head", block.Number(), hash[:4])
+ return nil
}
func (self *BlockChain) GasLimit() *big.Int {
@@ -181,6 +306,17 @@ func (self *BlockChain) LastBlockHash() common.Hash {
return self.currentBlock.Hash()
}
+// CurrentHeader retrieves the current head header of the canonical chain. The
+// header is retrieved from the blockchain's internal cache.
+func (self *BlockChain) CurrentHeader() *types.Header {
+ self.mu.RLock()
+ defer self.mu.RUnlock()
+
+ return self.currentHeader
+}
+
+// CurrentBlock retrieves the current head block of the canonical chain. The
+// block is retrieved from the blockchain's internal cache.
func (self *BlockChain) CurrentBlock() *types.Block {
self.mu.RLock()
defer self.mu.RUnlock()
@@ -188,11 +324,20 @@ func (self *BlockChain) CurrentBlock() *types.Block {
return self.currentBlock
}
+// CurrentFastBlock retrieves the current fast-sync head block of the canonical
+// chain. The block is retrieved from the blockchain's internal cache.
+func (self *BlockChain) CurrentFastBlock() *types.Block {
+ self.mu.RLock()
+ defer self.mu.RUnlock()
+
+ return self.currentFastBlock
+}
+
func (self *BlockChain) Status() (td *big.Int, currentBlock common.Hash, genesisBlock common.Hash) {
self.mu.RLock()
defer self.mu.RUnlock()
- return new(big.Int).Set(self.td), self.currentBlock.Hash(), self.genesisBlock.Hash()
+ return self.GetTd(self.currentBlock.Hash()), self.currentBlock.Hash(), self.genesisBlock.Hash()
}
func (self *BlockChain) SetProcessor(proc types.BlockProcessor) {
@@ -203,26 +348,6 @@ func (self *BlockChain) State() (*state.StateDB, error) {
return state.New(self.CurrentBlock().Root(), self.chainDb)
}
-func (bc *BlockChain) setLastState() error {
- head := GetHeadBlockHash(bc.chainDb)
- if head != (common.Hash{}) {
- block := bc.GetBlock(head)
- if block != nil {
- bc.currentBlock = block
- }
- } else {
- bc.Reset()
- }
- bc.td = bc.GetTd(bc.currentBlock.Hash())
- bc.currentGasLimit = CalcGasLimit(bc.currentBlock)
-
- if glog.V(logger.Info) {
- glog.Infof("Last block (#%v) %x TD=%v\n", bc.currentBlock.Number(), bc.currentBlock.Hash(), bc.td)
- }
-
- return nil
-}
-
// Reset purges the entire blockchain, restoring it to its genesis state.
func (bc *BlockChain) Reset() {
bc.ResetWithGenesisBlock(bc.genesisBlock)
@@ -231,20 +356,13 @@ func (bc *BlockChain) Reset() {
// ResetWithGenesisBlock purges the entire blockchain, restoring it to the
// specified genesis state.
func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) {
+ // Dump the entire block chain and purge the caches
+ bc.SetHead(0)
+
bc.mu.Lock()
defer bc.mu.Unlock()
- // Dump the entire block chain and purge the caches
- for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.ParentHash()) {
- DeleteBlock(bc.chainDb, block.Hash())
- }
- bc.headerCache.Purge()
- bc.bodyCache.Purge()
- bc.bodyRLPCache.Purge()
- bc.blockCache.Purge()
- bc.futureBlocks.Purge()
-
- // Prepare the genesis block and reinitialize the chain
+ // Prepare the genesis block and reinitialise the chain
if err := WriteTd(bc.chainDb, genesis.Hash(), genesis.Difficulty()); err != nil {
glog.Fatalf("failed to write genesis block TD: %v", err)
}
@@ -254,7 +372,8 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) {
bc.genesisBlock = genesis
bc.insert(bc.genesisBlock)
bc.currentBlock = bc.genesisBlock
- bc.setTotalDifficulty(genesis.Difficulty())
+ bc.currentHeader = bc.genesisBlock.Header()
+ bc.currentFastBlock = bc.genesisBlock
}
// Export writes the active chain to the given writer.
@@ -290,17 +409,30 @@ func (self *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error {
return nil
}
-// insert injects a block into the current chain block chain. Note, this function
-// assumes that the `mu` mutex is held!
+// insert injects a new head block into the current block chain. This method
+// assumes that the block is indeed a true head. It will also reset the head
+// header and the head fast sync block to this very same block to prevent them
+// from pointing to a possibly old canonical chain (i.e. side chain by now).
+//
+// Note, this function assumes that the `mu` mutex is held!
func (bc *BlockChain) insert(block *types.Block) {
// Add the block to the canonical chain number scheme and mark as the head
if err := WriteCanonicalHash(bc.chainDb, block.Hash(), block.NumberU64()); err != nil {
glog.Fatalf("failed to insert block number: %v", err)
}
if err := WriteHeadBlockHash(bc.chainDb, block.Hash()); err != nil {
- glog.Fatalf("failed to insert block number: %v", err)
+ glog.Fatalf("failed to insert head block hash: %v", err)
}
+ if err := WriteHeadHeaderHash(bc.chainDb, block.Hash()); err != nil {
+ glog.Fatalf("failed to insert head header hash: %v", err)
+ }
+ if err := WriteHeadFastBlockHash(bc.chainDb, block.Hash()); err != nil {
+ glog.Fatalf("failed to insert head fast block hash: %v", err)
+ }
+ // Update the internal state with the head block
bc.currentBlock = block
+ bc.currentHeader = block.Header()
+ bc.currentFastBlock = block
}
// Accessors
@@ -456,19 +588,15 @@ func (self *BlockChain) GetBlocksFromHash(hash common.Hash, n int) (blocks []*ty
return
}
-func (self *BlockChain) GetUnclesInChain(block *types.Block, length int) (uncles []*types.Header) {
+// GetUnclesInChain retrieves all the uncles from a given block backwards until
+// a specific distance is reached.
+func (self *BlockChain) GetUnclesInChain(block *types.Block, length int) []*types.Header {
+ uncles := []*types.Header{}
for i := 0; block != nil && i < length; i++ {
uncles = append(uncles, block.Uncles()...)
block = self.GetBlock(block.ParentHash())
}
-
- return
-}
-
-// setTotalDifficulty updates the TD of the chain manager. Note, this function
-// assumes that the `mu` mutex is held!
-func (bc *BlockChain) setTotalDifficulty(td *big.Int) {
- bc.td = new(big.Int).Set(td)
+ return uncles
}
func (bc *BlockChain) Stop() {
@@ -504,6 +632,337 @@ const (
SideStatTy
)
+// writeHeader writes a header into the local chain, given that its parent is
+// already known. If the total difficulty of the newly inserted header becomes
+// greater than the current known TD, the canonical chain is re-routed.
+//
+// Note: This method is not concurrent-safe with inserting blocks simultaneously
+// into the chain, as side effects caused by reorganisations cannot be emulated
+// without the real blocks. Hence, writing headers directly should only be done
+// in two scenarios: pure-header mode of operation (light clients), or properly
+// separated header/block phases (non-archive clients).
+func (self *BlockChain) writeHeader(header *types.Header) error {
+ self.wg.Add(1)
+ defer self.wg.Done()
+
+ // Calculate the total difficulty of the header
+ ptd := self.GetTd(header.ParentHash)
+ if ptd == nil {
+ return ParentError(header.ParentHash)
+ }
+ td := new(big.Int).Add(header.Difficulty, ptd)
+
+ // Make sure no inconsistent state is leaked during insertion
+ self.mu.Lock()
+ defer self.mu.Unlock()
+
+ // If the total difficulty is higher than our known, add it to the canonical chain
+ if td.Cmp(self.GetTd(self.currentHeader.Hash())) > 0 {
+ // Delete any canonical number assignments above the new head
+ for i := header.Number.Uint64() + 1; GetCanonicalHash(self.chainDb, i) != (common.Hash{}); i++ {
+ DeleteCanonicalHash(self.chainDb, i)
+ }
+ // Overwrite any stale canonical number assignments
+ head := self.GetHeader(header.ParentHash)
+ for GetCanonicalHash(self.chainDb, head.Number.Uint64()) != head.Hash() {
+ WriteCanonicalHash(self.chainDb, head.Hash(), head.Number.Uint64())
+ head = self.GetHeader(head.ParentHash)
+ }
+ // Extend the canonical chain with the new header
+ if err := WriteCanonicalHash(self.chainDb, header.Hash(), header.Number.Uint64()); err != nil {
+ glog.Fatalf("failed to insert header number: %v", err)
+ }
+ if err := WriteHeadHeaderHash(self.chainDb, header.Hash()); err != nil {
+ glog.Fatalf("failed to insert head header hash: %v", err)
+ }
+ self.currentHeader = types.CopyHeader(header)
+ }
+ // Irrelevant of the canonical status, write the header itself to the database
+ if err := WriteTd(self.chainDb, header.Hash(), td); err != nil {
+ glog.Fatalf("failed to write header total difficulty: %v", err)
+ }
+ if err := WriteHeader(self.chainDb, header); err != nil {
+ glog.Fatalf("filed to write header contents: %v", err)
+ }
+ return nil
+}
+
+// InsertHeaderChain attempts to insert the given header chain in to the local
+// chain, possibly creating a reorg. If an error is returned, it will return the
+// index number of the failing header as well an error describing what went wrong.
+//
+// The verify parameter can be used to fine tune whether nonce verification
+// should be done or not. The reason behind the optional check is because some
+// of the header retrieval mechanisms already need to verfy nonces, as well as
+// because nonces can be verified sparsely, not needing to check each.
+func (self *BlockChain) InsertHeaderChain(chain []*types.Header, checkFreq int) (int, error) {
+ self.wg.Add(1)
+ defer self.wg.Done()
+
+ // Make sure only one thread manipulates the chain at once
+ self.chainmu.Lock()
+ defer self.chainmu.Unlock()
+
+ // Collect some import statistics to report on
+ stats := struct{ processed, ignored int }{}
+ start := time.Now()
+
+ // Generate the list of headers that should be POW verified
+ verify := make([]bool, len(chain))
+ for i := 0; i < len(verify)/checkFreq; i++ {
+ index := i*checkFreq + self.rand.Intn(checkFreq)
+ if index >= len(verify) {
+ index = len(verify) - 1
+ }
+ verify[index] = true
+ }
+ verify[len(verify)-1] = true // Last should always be verified to avoid junk
+
+ // Create the header verification task queue and worker functions
+ tasks := make(chan int, len(chain))
+ for i := 0; i < len(chain); i++ {
+ tasks <- i
+ }
+ close(tasks)
+
+ errs, failed := make([]error, len(tasks)), int32(0)
+ process := func(worker int) {
+ for index := range tasks {
+ header, hash := chain[index], chain[index].Hash()
+
+ // Short circuit insertion if shutting down or processing failed
+ if atomic.LoadInt32(&self.procInterrupt) == 1 {
+ return
+ }
+ if atomic.LoadInt32(&failed) > 0 {
+ return
+ }
+ // Short circuit if the header is bad or already known
+ if BadHashes[hash] {
+ errs[index] = BadHashError(hash)
+ atomic.AddInt32(&failed, 1)
+ return
+ }
+ if self.HasHeader(hash) {
+ continue
+ }
+ // Verify that the header honors the chain parameters
+ checkPow := verify[index]
+
+ var err error
+ if index == 0 {
+ err = self.processor.ValidateHeader(header, checkPow, false)
+ } else {
+ err = self.processor.ValidateHeaderWithParent(header, chain[index-1], checkPow, false)
+ }
+ if err != nil {
+ errs[index] = err
+ atomic.AddInt32(&failed, 1)
+ return
+ }
+ }
+ }
+ // Start as many worker threads as goroutines allowed
+ pending := new(sync.WaitGroup)
+ for i := 0; i < runtime.GOMAXPROCS(0); i++ {
+ pending.Add(1)
+ go func(id int) {
+ defer pending.Done()
+ process(id)
+ }(i)
+ }
+ pending.Wait()
+
+ // If anything failed, report
+ if failed > 0 {
+ for i, err := range errs {
+ if err != nil {
+ return i, err
+ }
+ }
+ }
+ // All headers passed verification, import them into the database
+ for i, header := range chain {
+ // Short circuit insertion if shutting down
+ if atomic.LoadInt32(&self.procInterrupt) == 1 {
+ glog.V(logger.Debug).Infoln("premature abort during header chain processing")
+ break
+ }
+ hash := header.Hash()
+
+ // If the header's already known, skip it, otherwise store
+ if self.HasHeader(hash) {
+ stats.ignored++
+ continue
+ }
+ if err := self.writeHeader(header); err != nil {
+ return i, err
+ }
+ stats.processed++
+ }
+ // Report some public statistics so the user has a clue what's going on
+ first, last := chain[0], chain[len(chain)-1]
+ glog.V(logger.Info).Infof("imported %d header(s) (%d ignored) in %v. #%v [%x… / %x…]", stats.processed, stats.ignored,
+ time.Since(start), last.Number, first.Hash().Bytes()[:4], last.Hash().Bytes()[:4])
+
+ return 0, nil
+}
+
+// Rollback is designed to remove a chain of links from the database that aren't
+// certain enough to be valid.
+func (self *BlockChain) Rollback(chain []common.Hash) {
+ self.mu.Lock()
+ defer self.mu.Unlock()
+
+ for i := len(chain) - 1; i >= 0; i-- {
+ hash := chain[i]
+
+ if self.currentHeader.Hash() == hash {
+ self.currentHeader = self.GetHeader(self.currentHeader.ParentHash)
+ WriteHeadHeaderHash(self.chainDb, self.currentHeader.Hash())
+ }
+ if self.currentFastBlock.Hash() == hash {
+ self.currentFastBlock = self.GetBlock(self.currentFastBlock.ParentHash())
+ WriteHeadFastBlockHash(self.chainDb, self.currentFastBlock.Hash())
+ }
+ if self.currentBlock.Hash() == hash {
+ self.currentBlock = self.GetBlock(self.currentBlock.ParentHash())
+ WriteHeadBlockHash(self.chainDb, self.currentBlock.Hash())
+ }
+ }
+}
+
+// InsertReceiptChain attempts to complete an already existing header chain with
+// transaction and receipt data.
+func (self *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) {
+ self.wg.Add(1)
+ defer self.wg.Done()
+
+ // Collect some import statistics to report on
+ stats := struct{ processed, ignored int32 }{}
+ start := time.Now()
+
+ // Create the block importing task queue and worker functions
+ tasks := make(chan int, len(blockChain))
+ for i := 0; i < len(blockChain) && i < len(receiptChain); i++ {
+ tasks <- i
+ }
+ close(tasks)
+
+ errs, failed := make([]error, len(tasks)), int32(0)
+ process := func(worker int) {
+ for index := range tasks {
+ block, receipts := blockChain[index], receiptChain[index]
+
+ // Short circuit insertion if shutting down or processing failed
+ if atomic.LoadInt32(&self.procInterrupt) == 1 {
+ return
+ }
+ if atomic.LoadInt32(&failed) > 0 {
+ return
+ }
+ // Short circuit if the owner header is unknown
+ if !self.HasHeader(block.Hash()) {
+ errs[index] = fmt.Errorf("containing header #%d [%x…] unknown", block.Number(), block.Hash().Bytes()[:4])
+ atomic.AddInt32(&failed, 1)
+ return
+ }
+ // Skip if the entire data is already known
+ if self.HasBlock(block.Hash()) {
+ atomic.AddInt32(&stats.ignored, 1)
+ continue
+ }
+ // Compute all the non-consensus fields of the receipts
+ transactions, logIndex := block.Transactions(), uint(0)
+ for j := 0; j < len(receipts); j++ {
+ // The transaction hash can be retrieved from the transaction itself
+ receipts[j].TxHash = transactions[j].Hash()
+
+ // The contract address can be derived from the transaction itself
+ if MessageCreatesContract(transactions[j]) {
+ from, _ := transactions[j].From()
+ receipts[j].ContractAddress = crypto.CreateAddress(from, transactions[j].Nonce())
+ }
+ // The used gas can be calculated based on previous receipts
+ if j == 0 {
+ receipts[j].GasUsed = new(big.Int).Set(receipts[j].CumulativeGasUsed)
+ } else {
+ receipts[j].GasUsed = new(big.Int).Sub(receipts[j].CumulativeGasUsed, receipts[j-1].CumulativeGasUsed)
+ }
+ // The derived log fields can simply be set from the block and transaction
+ for k := 0; k < len(receipts[j].Logs); k++ {
+ receipts[j].Logs[k].BlockNumber = block.NumberU64()
+ receipts[j].Logs[k].BlockHash = block.Hash()
+ receipts[j].Logs[k].TxHash = receipts[j].TxHash
+ receipts[j].Logs[k].TxIndex = uint(j)
+ receipts[j].Logs[k].Index = logIndex
+ logIndex++
+ }
+ }
+ // Write all the data out into the database
+ if err := WriteBody(self.chainDb, block.Hash(), &types.Body{block.Transactions(), block.Uncles()}); err != nil {
+ errs[index] = fmt.Errorf("failed to write block body: %v", err)
+ atomic.AddInt32(&failed, 1)
+ glog.Fatal(errs[index])
+ return
+ }
+ if err := PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
+ errs[index] = fmt.Errorf("failed to write block receipts: %v", err)
+ atomic.AddInt32(&failed, 1)
+ glog.Fatal(errs[index])
+ return
+ }
+ if err := WriteMipmapBloom(self.chainDb, block.NumberU64(), receipts); err != nil {
+ errs[index] = fmt.Errorf("failed to write log blooms: %v", err)
+ atomic.AddInt32(&failed, 1)
+ glog.Fatal(errs[index])
+ return
+ }
+ atomic.AddInt32(&stats.processed, 1)
+ }
+ }
+ // Start as many worker threads as goroutines allowed
+ pending := new(sync.WaitGroup)
+ for i := 0; i < runtime.GOMAXPROCS(0); i++ {
+ pending.Add(1)
+ go func(id int) {
+ defer pending.Done()
+ process(id)
+ }(i)
+ }
+ pending.Wait()
+
+ // If anything failed, report
+ if failed > 0 {
+ for i, err := range errs {
+ if err != nil {
+ return i, err
+ }
+ }
+ }
+ if atomic.LoadInt32(&self.procInterrupt) == 1 {
+ glog.V(logger.Debug).Infoln("premature abort during receipt chain processing")
+ return 0, nil
+ }
+ // Update the head fast sync block if better
+ self.mu.Lock()
+ head := blockChain[len(errs)-1]
+ if self.GetTd(self.currentFastBlock.Hash()).Cmp(self.GetTd(head.Hash())) < 0 {
+ if err := WriteHeadFastBlockHash(self.chainDb, head.Hash()); err != nil {
+ glog.Fatalf("failed to update head fast block hash: %v", err)
+ }
+ self.currentFastBlock = head
+ }
+ self.mu.Unlock()
+
+ // Report some public statistics so the user has a clue what's going on
+ first, last := blockChain[0], blockChain[len(blockChain)-1]
+ glog.V(logger.Info).Infof("imported %d receipt(s) (%d ignored) in %v. #%d [%x… / %x…]", stats.processed, stats.ignored,
+ time.Since(start), last.Number(), first.Hash().Bytes()[:4], last.Hash().Bytes()[:4])
+
+ return 0, nil
+}
+
// WriteBlock writes the block to the chain.
func (self *BlockChain) WriteBlock(block *types.Block) (status writeStatus, err error) {
self.wg.Add(1)
@@ -516,38 +975,31 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status writeStatus, err
}
td := new(big.Int).Add(block.Difficulty(), ptd)
- self.mu.RLock()
- cblock := self.currentBlock
- self.mu.RUnlock()
+ // Make sure no inconsistent state is leaked during insertion
+ self.mu.Lock()
+ defer self.mu.Unlock()
- // Compare the TD of the last known block in the canonical chain to make sure it's greater.
- // At this point it's possible that a different chain (fork) becomes the new canonical chain.
- if td.Cmp(self.Td()) > 0 {
- // chain fork
- if block.ParentHash() != cblock.Hash() {
- // during split we merge two different chains and create the new canonical chain
- err := self.reorg(cblock, block)
- if err != nil {
+ // If the total difficulty is higher than our known, add it to the canonical chain
+ if td.Cmp(self.GetTd(self.currentBlock.Hash())) > 0 {
+ // Reorganize the chain if the parent is not the head block
+ if block.ParentHash() != self.currentBlock.Hash() {
+ if err := self.reorg(self.currentBlock, block); err != nil {
return NonStatTy, err
}
}
- status = CanonStatTy
-
- self.mu.Lock()
- self.setTotalDifficulty(td)
+ // Insert the block as the new head of the chain
self.insert(block)
- self.mu.Unlock()
+ status = CanonStatTy
} else {
status = SideStatTy
}
-
+ // Irrelevant of the canonical status, write the block itself to the database
if err := WriteTd(self.chainDb, block.Hash(), td); err != nil {
glog.Fatalf("failed to write block total difficulty: %v", err)
}
if err := WriteBlock(self.chainDb, block); err != nil {
glog.Fatalf("filed to write block contents: %v", err)
}
- // Delete from future blocks
self.futureBlocks.Remove(block.Hash())
return
@@ -580,7 +1032,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
txcount := 0
for i, block := range chain {
if atomic.LoadInt32(&self.procInterrupt) == 1 {
- glog.V(logger.Debug).Infoln("Premature abort during chain processing")
+ glog.V(logger.Debug).Infoln("Premature abort during block chain processing")
break
}
@@ -636,7 +1088,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
return i, err
}
- if err := PutBlockReceipts(self.chainDb, block, receipts); err != nil {
+ if err := PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
glog.V(logger.Warn).Infoln("error writing block receipts:", err)
}
@@ -691,9 +1143,6 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (int, error) {
// to be part of the new canonical chain and accumulates potential missing transactions and post an
// event about them
func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error {
- self.mu.Lock()
- defer self.mu.Unlock()
-
var (
newChain types.Blocks
commonBlock *types.Block
@@ -788,8 +1237,7 @@ func (self *BlockChain) postChainEvents(events []interface{}) {
if event, ok := event.(ChainEvent); ok {
// We need some control over the mining operation. Acquiring locks and waiting for the miner to create new block takes too long
// and in most cases isn't even necessary.
- if self.currentBlock.Hash() == event.Hash {
- self.currentGasLimit = CalcGasLimit(event.Block)
+ if self.LastBlockHash() == event.Hash {
self.eventMux.Post(ChainHeadEvent{event.Block})
}
}
diff --git a/core/blockchain_test.go b/core/blockchain_test.go
index 13971ccba..8ddc5032b 100644
--- a/core/blockchain_test.go
+++ b/core/blockchain_test.go
@@ -64,44 +64,58 @@ func theBlockChain(db ethdb.Database, t *testing.T) *BlockChain {
}
// Test fork of length N starting from block i
-func testFork(t *testing.T, bman *BlockProcessor, i, N int, f func(td1, td2 *big.Int)) {
- // switch databases to process the new chain
- db, err := ethdb.NewMemDatabase()
- if err != nil {
- t.Fatal("Failed to create db:", err)
- }
- // copy old chain up to i into new db with deterministic canonical
- bman2, err := newCanonical(i, db)
+func testFork(t *testing.T, processor *BlockProcessor, i, n int, full bool, comparator func(td1, td2 *big.Int)) {
+ // Copy old chain up to #i into a new db
+ db, processor2, err := newCanonical(i, full)
if err != nil {
t.Fatal("could not make new canonical in testFork", err)
}
- // assert the bmans have the same block at i
- bi1 := bman.bc.GetBlockByNumber(uint64(i)).Hash()
- bi2 := bman2.bc.GetBlockByNumber(uint64(i)).Hash()
- if bi1 != bi2 {
- fmt.Printf("%+v\n%+v\n\n", bi1, bi2)
- t.Fatal("chains do not have the same hash at height", i)
+ // Assert the chains have the same header/block at #i
+ var hash1, hash2 common.Hash
+ if full {
+ hash1 = processor.bc.GetBlockByNumber(uint64(i)).Hash()
+ hash2 = processor2.bc.GetBlockByNumber(uint64(i)).Hash()
+ } else {
+ hash1 = processor.bc.GetHeaderByNumber(uint64(i)).Hash()
+ hash2 = processor2.bc.GetHeaderByNumber(uint64(i)).Hash()
}
- bman2.bc.SetProcessor(bman2)
-
- // extend the fork
- parent := bman2.bc.CurrentBlock()
- chainB := makeChain(parent, N, db, forkSeed)
- _, err = bman2.bc.InsertChain(chainB)
- if err != nil {
- t.Fatal("Insert chain error for fork:", err)
+ if hash1 != hash2 {
+ t.Errorf("chain content mismatch at %d: have hash %v, want hash %v", i, hash2, hash1)
}
-
- tdpre := bman.bc.Td()
- // Test the fork's blocks on the original chain
- td, err := testChain(chainB, bman)
- if err != nil {
- t.Fatal("expected chainB not to give errors:", err)
+ // Extend the newly created chain
+ var (
+ blockChainB []*types.Block
+ headerChainB []*types.Header
+ )
+ if full {
+ blockChainB = makeBlockChain(processor2.bc.CurrentBlock(), n, db, forkSeed)
+ if _, err := processor2.bc.InsertChain(blockChainB); err != nil {
+ t.Fatalf("failed to insert forking chain: %v", err)
+ }
+ } else {
+ headerChainB = makeHeaderChain(processor2.bc.CurrentHeader(), n, db, forkSeed)
+ if _, err := processor2.bc.InsertHeaderChain(headerChainB, 1); err != nil {
+ t.Fatalf("failed to insert forking chain: %v", err)
+ }
}
- // Compare difficulties
- f(tdpre, td)
+ // Sanity check that the forked chain can be imported into the original
+ var tdPre, tdPost *big.Int
- // Loop over parents making sure reconstruction is done properly
+ if full {
+ tdPre = processor.bc.GetTd(processor.bc.CurrentBlock().Hash())
+ if err := testBlockChainImport(blockChainB, processor); err != nil {
+ t.Fatalf("failed to import forked block chain: %v", err)
+ }
+ tdPost = processor.bc.GetTd(blockChainB[len(blockChainB)-1].Hash())
+ } else {
+ tdPre = processor.bc.GetTd(processor.bc.CurrentHeader().Hash())
+ if err := testHeaderChainImport(headerChainB, processor); err != nil {
+ t.Fatalf("failed to import forked header chain: %v", err)
+ }
+ tdPost = processor.bc.GetTd(headerChainB[len(headerChainB)-1].Hash())
+ }
+ // Compare the total difficulties of the chains
+ comparator(tdPre, tdPost)
}
func printChain(bc *BlockChain) {
@@ -111,22 +125,41 @@ func printChain(bc *BlockChain) {
}
}
-// process blocks against a chain
-func testChain(chainB types.Blocks, bman *BlockProcessor) (*big.Int, error) {
- for _, block := range chainB {
- _, _, err := bman.bc.processor.Process(block)
- if err != nil {
+// testBlockChainImport tries to process a chain of blocks, writing them into
+// the database if successful.
+func testBlockChainImport(chain []*types.Block, processor *BlockProcessor) error {
+ for _, block := range chain {
+ // Try and process the block
+ if _, _, err := processor.Process(block); err != nil {
if IsKnownBlockErr(err) {
continue
}
- return nil, err
+ return err
}
- bman.bc.mu.Lock()
- WriteTd(bman.bc.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), bman.bc.GetTd(block.ParentHash())))
- WriteBlock(bman.bc.chainDb, block)
- bman.bc.mu.Unlock()
+ // Manually insert the block into the database, but don't reorganize (allows subsequent testing)
+ processor.bc.mu.Lock()
+ WriteTd(processor.chainDb, block.Hash(), new(big.Int).Add(block.Difficulty(), processor.bc.GetTd(block.ParentHash())))
+ WriteBlock(processor.chainDb, block)
+ processor.bc.mu.Unlock()
}
- return bman.bc.GetTd(chainB[len(chainB)-1].Hash()), nil
+ return nil
+}
+
+// testHeaderChainImport tries to process a chain of header, writing them into
+// the database if successful.
+func testHeaderChainImport(chain []*types.Header, processor *BlockProcessor) error {
+ for _, header := range chain {
+ // Try and validate the header
+ if err := processor.ValidateHeader(header, false, false); err != nil {
+ return err
+ }
+ // Manually insert the header into the database, but don't reorganize (allows subsequent testing)
+ processor.bc.mu.Lock()
+ WriteTd(processor.chainDb, header.Hash(), new(big.Int).Add(header.Difficulty, processor.bc.GetTd(header.ParentHash)))
+ WriteHeader(processor.chainDb, header)
+ processor.bc.mu.Unlock()
+ }
+ return nil
}
func loadChain(fn string, t *testing.T) (types.Blocks, error) {
@@ -154,139 +187,147 @@ func insertChain(done chan bool, blockchain *BlockChain, chain types.Blocks, t *
}
func TestLastBlock(t *testing.T) {
- db, err := ethdb.NewMemDatabase()
- if err != nil {
- t.Fatal("Failed to create db:", err)
- }
+ db, _ := ethdb.NewMemDatabase()
+
bchain := theBlockChain(db, t)
- block := makeChain(bchain.CurrentBlock(), 1, db, 0)[0]
+ block := makeBlockChain(bchain.CurrentBlock(), 1, db, 0)[0]
bchain.insert(block)
if block.Hash() != GetHeadBlockHash(db) {
t.Errorf("Write/Get HeadBlockHash failed")
}
}
-func TestExtendCanonical(t *testing.T) {
- CanonicalLength := 5
- db, err := ethdb.NewMemDatabase()
+// Tests that given a starting canonical chain of a given size, it can be extended
+// with various length chains.
+func TestExtendCanonicalHeaders(t *testing.T) { testExtendCanonical(t, false) }
+func TestExtendCanonicalBlocks(t *testing.T) { testExtendCanonical(t, true) }
+
+func testExtendCanonical(t *testing.T, full bool) {
+ length := 5
+
+ // Make first chain starting from genesis
+ _, processor, err := newCanonical(length, full)
if err != nil {
- t.Fatal("Failed to create db:", err)
+ t.Fatalf("failed to make new canonical chain: %v", err)
}
- // make first chain starting from genesis
- bman, err := newCanonical(CanonicalLength, db)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- f := func(td1, td2 *big.Int) {
+ // Define the difficulty comparator
+ better := func(td1, td2 *big.Int) {
if td2.Cmp(td1) <= 0 {
- t.Error("expected chainB to have higher difficulty. Got", td2, "expected more than", td1)
+ t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1)
}
}
- // Start fork from current height (CanonicalLength)
- testFork(t, bman, CanonicalLength, 1, f)
- testFork(t, bman, CanonicalLength, 2, f)
- testFork(t, bman, CanonicalLength, 5, f)
- testFork(t, bman, CanonicalLength, 10, f)
+ // Start fork from current height
+ testFork(t, processor, length, 1, full, better)
+ testFork(t, processor, length, 2, full, better)
+ testFork(t, processor, length, 5, full, better)
+ testFork(t, processor, length, 10, full, better)
}
-func TestShorterFork(t *testing.T) {
- db, err := ethdb.NewMemDatabase()
+// Tests that given a starting canonical chain of a given size, creating shorter
+// forks do not take canonical ownership.
+func TestShorterForkHeaders(t *testing.T) { testShorterFork(t, false) }
+func TestShorterForkBlocks(t *testing.T) { testShorterFork(t, true) }
+
+func testShorterFork(t *testing.T, full bool) {
+ length := 10
+
+ // Make first chain starting from genesis
+ _, processor, err := newCanonical(length, full)
if err != nil {
- t.Fatal("Failed to create db:", err)
+ t.Fatalf("failed to make new canonical chain: %v", err)
}
- // make first chain starting from genesis
- bman, err := newCanonical(10, db)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- f := func(td1, td2 *big.Int) {
+ // Define the difficulty comparator
+ worse := func(td1, td2 *big.Int) {
if td2.Cmp(td1) >= 0 {
- t.Error("expected chainB to have lower difficulty. Got", td2, "expected less than", td1)
+ t.Errorf("total difficulty mismatch: have %v, expected less than %v", td2, td1)
}
}
- // Sum of numbers must be less than 10
- // for this to be a shorter fork
- testFork(t, bman, 0, 3, f)
- testFork(t, bman, 0, 7, f)
- testFork(t, bman, 1, 1, f)
- testFork(t, bman, 1, 7, f)
- testFork(t, bman, 5, 3, f)
- testFork(t, bman, 5, 4, f)
+ // Sum of numbers must be less than `length` for this to be a shorter fork
+ testFork(t, processor, 0, 3, full, worse)
+ testFork(t, processor, 0, 7, full, worse)
+ testFork(t, processor, 1, 1, full, worse)
+ testFork(t, processor, 1, 7, full, worse)
+ testFork(t, processor, 5, 3, full, worse)
+ testFork(t, processor, 5, 4, full, worse)
}
-func TestLongerFork(t *testing.T) {
- db, err := ethdb.NewMemDatabase()
+// Tests that given a starting canonical chain of a given size, creating longer
+// forks do take canonical ownership.
+func TestLongerForkHeaders(t *testing.T) { testLongerFork(t, false) }
+func TestLongerForkBlocks(t *testing.T) { testLongerFork(t, true) }
+
+func testLongerFork(t *testing.T, full bool) {
+ length := 10
+
+ // Make first chain starting from genesis
+ _, processor, err := newCanonical(length, full)
if err != nil {
- t.Fatal("Failed to create db:", err)
+ t.Fatalf("failed to make new canonical chain: %v", err)
}
- // make first chain starting from genesis
- bman, err := newCanonical(10, db)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- f := func(td1, td2 *big.Int) {
+ // Define the difficulty comparator
+ better := func(td1, td2 *big.Int) {
if td2.Cmp(td1) <= 0 {
- t.Error("expected chainB to have higher difficulty. Got", td2, "expected more than", td1)
+ t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1)
}
}
- // Sum of numbers must be greater than 10
- // for this to be a longer fork
- testFork(t, bman, 0, 11, f)
- testFork(t, bman, 0, 15, f)
- testFork(t, bman, 1, 10, f)
- testFork(t, bman, 1, 12, f)
- testFork(t, bman, 5, 6, f)
- testFork(t, bman, 5, 8, f)
+ // Sum of numbers must be greater than `length` for this to be a longer fork
+ testFork(t, processor, 0, 11, full, better)
+ testFork(t, processor, 0, 15, full, better)
+ testFork(t, processor, 1, 10, full, better)
+ testFork(t, processor, 1, 12, full, better)
+ testFork(t, processor, 5, 6, full, better)
+ testFork(t, processor, 5, 8, full, better)
}
-func TestEqualFork(t *testing.T) {
- db, err := ethdb.NewMemDatabase()
+// Tests that given a starting canonical chain of a given size, creating equal
+// forks do take canonical ownership.
+func TestEqualForkHeaders(t *testing.T) { testEqualFork(t, false) }
+func TestEqualForkBlocks(t *testing.T) { testEqualFork(t, true) }
+
+func testEqualFork(t *testing.T, full bool) {
+ length := 10
+
+ // Make first chain starting from genesis
+ _, processor, err := newCanonical(length, full)
if err != nil {
- t.Fatal("Failed to create db:", err)
+ t.Fatalf("failed to make new canonical chain: %v", err)
}
- bman, err := newCanonical(10, db)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- f := func(td1, td2 *big.Int) {
+ // Define the difficulty comparator
+ equal := func(td1, td2 *big.Int) {
if td2.Cmp(td1) != 0 {
- t.Error("expected chainB to have equal difficulty. Got", td2, "expected ", td1)
+ t.Errorf("total difficulty mismatch: have %v, want %v", td2, td1)
}
}
- // Sum of numbers must be equal to 10
- // for this to be an equal fork
- testFork(t, bman, 0, 10, f)
- testFork(t, bman, 1, 9, f)
- testFork(t, bman, 2, 8, f)
- testFork(t, bman, 5, 5, f)
- testFork(t, bman, 6, 4, f)
- testFork(t, bman, 9, 1, f)
+ // Sum of numbers must be equal to `length` for this to be an equal fork
+ testFork(t, processor, 0, 10, full, equal)
+ testFork(t, processor, 1, 9, full, equal)
+ testFork(t, processor, 2, 8, full, equal)
+ testFork(t, processor, 5, 5, full, equal)
+ testFork(t, processor, 6, 4, full, equal)
+ testFork(t, processor, 9, 1, full, equal)
}
-func TestBrokenChain(t *testing.T) {
- db, err := ethdb.NewMemDatabase()
+// Tests that chains missing links do not get accepted by the processor.
+func TestBrokenHeaderChain(t *testing.T) { testBrokenChain(t, false) }
+func TestBrokenBlockChain(t *testing.T) { testBrokenChain(t, true) }
+
+func testBrokenChain(t *testing.T, full bool) {
+ // Make chain starting from genesis
+ db, processor, err := newCanonical(10, full)
if err != nil {
- t.Fatal("Failed to create db:", err)
+ t.Fatalf("failed to make new canonical chain: %v", err)
}
- bman, err := newCanonical(10, db)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- db2, err := ethdb.NewMemDatabase()
- if err != nil {
- t.Fatal("Failed to create db:", err)
- }
- bman2, err := newCanonical(10, db2)
- if err != nil {
- t.Fatal("Could not make new canonical chain:", err)
- }
- bman2.bc.SetProcessor(bman2)
- parent := bman2.bc.CurrentBlock()
- chainB := makeChain(parent, 5, db2, forkSeed)
- chainB = chainB[1:]
- _, err = testChain(chainB, bman)
- if err == nil {
- t.Error("expected broken chain to return error")
+ // Create a forked chain, and try to insert with a missing link
+ if full {
+ chain := makeBlockChain(processor.bc.CurrentBlock(), 5, db, forkSeed)[1:]
+ if err := testBlockChainImport(chain, processor); err == nil {
+ t.Errorf("broken block chain not reported")
+ }
+ } else {
+ chain := makeHeaderChain(processor.bc.CurrentHeader(), 5, db, forkSeed)[1:]
+ if err := testHeaderChainImport(chain, processor); err == nil {
+ t.Errorf("broken header chain not reported")
+ }
}
}
@@ -374,15 +415,29 @@ func TestChainMultipleInsertions(t *testing.T) {
type bproc struct{}
-func (bproc) Process(*types.Block) (vm.Logs, types.Receipts, error) { return nil, nil, nil }
+func (bproc) Process(*types.Block) (vm.Logs, types.Receipts, error) { return nil, nil, nil }
+func (bproc) ValidateHeader(*types.Header, bool, bool) error { return nil }
+func (bproc) ValidateHeaderWithParent(*types.Header, *types.Header, bool, bool) error { return nil }
-func makeChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
+func makeHeaderChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Header {
+ blocks := makeBlockChainWithDiff(genesis, d, seed)
+ headers := make([]*types.Header, len(blocks))
+ for i, block := range blocks {
+ headers[i] = block.Header()
+ }
+ return headers
+}
+
+func makeBlockChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block {
var chain []*types.Block
for i, difficulty := range d {
header := &types.Header{
- Coinbase: common.Address{seed},
- Number: big.NewInt(int64(i + 1)),
- Difficulty: big.NewInt(int64(difficulty)),
+ Coinbase: common.Address{seed},
+ Number: big.NewInt(int64(i + 1)),
+ Difficulty: big.NewInt(int64(difficulty)),
+ UncleHash: types.EmptyUncleHash,
+ TxHash: types.EmptyRootHash,
+ ReceiptHash: types.EmptyRootHash,
}
if i == 0 {
header.ParentHash = genesis.Hash()
@@ -397,7 +452,7 @@ func makeChainWithDiff(genesis *types.Block, d []int, seed byte) []*types.Block
func chm(genesis *types.Block, db ethdb.Database) *BlockChain {
var eventMux event.TypeMux
- bc := &BlockChain{chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}}
+ bc := &BlockChain{chainDb: db, genesisBlock: genesis, eventMux: &eventMux, pow: FakePow{}, rand: rand.New(rand.NewSource(0))}
bc.headerCache, _ = lru.New(100)
bc.bodyCache, _ = lru.New(100)
bc.bodyRLPCache, _ = lru.New(100)
@@ -410,147 +465,381 @@ func chm(genesis *types.Block, db ethdb.Database) *BlockChain {
return bc
}
-func TestReorgLongest(t *testing.T) {
- db, _ := ethdb.NewMemDatabase()
+// Tests that reorganizing a long difficult chain after a short easy one
+// overwrites the canonical numbers and links in the database.
+func TestReorgLongHeaders(t *testing.T) { testReorgLong(t, false) }
+func TestReorgLongBlocks(t *testing.T) { testReorgLong(t, true) }
- genesis, err := WriteTestNetGenesisBlock(db, 0)
- if err != nil {
- t.Error(err)
- t.FailNow()
- }
+func testReorgLong(t *testing.T, full bool) {
+ testReorg(t, []int{1, 2, 4}, []int{1, 2, 3, 4}, 10, full)
+}
+
+// Tests that reorganizing a short difficult chain after a long easy one
+// overwrites the canonical numbers and links in the database.
+func TestReorgShortHeaders(t *testing.T) { testReorgShort(t, false) }
+func TestReorgShortBlocks(t *testing.T) { testReorgShort(t, true) }
+
+func testReorgShort(t *testing.T, full bool) {
+ testReorg(t, []int{1, 2, 3, 4}, []int{1, 10}, 11, full)
+}
+
+func testReorg(t *testing.T, first, second []int, td int64, full bool) {
+ // Create a pristine block chain
+ db, _ := ethdb.NewMemDatabase()
+ genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
- chain1 := makeChainWithDiff(genesis, []int{1, 2, 4}, 10)
- chain2 := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 11)
-
- bc.InsertChain(chain1)
- bc.InsertChain(chain2)
-
- prev := bc.CurrentBlock()
- for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
- if prev.ParentHash() != block.Hash() {
- t.Errorf("parent hash mismatch %x - %x", prev.ParentHash(), block.Hash())
+ // Insert an easy and a difficult chain afterwards
+ if full {
+ bc.InsertChain(makeBlockChainWithDiff(genesis, first, 11))
+ bc.InsertChain(makeBlockChainWithDiff(genesis, second, 22))
+ } else {
+ bc.InsertHeaderChain(makeHeaderChainWithDiff(genesis, first, 11), 1)
+ bc.InsertHeaderChain(makeHeaderChainWithDiff(genesis, second, 22), 1)
+ }
+ // Check that the chain is valid number and link wise
+ if full {
+ prev := bc.CurrentBlock()
+ for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
+ if prev.ParentHash() != block.Hash() {
+ t.Errorf("parent block hash mismatch: have %x, want %x", prev.ParentHash(), block.Hash())
+ }
+ }
+ } else {
+ prev := bc.CurrentHeader()
+ for header := bc.GetHeaderByNumber(bc.CurrentHeader().Number.Uint64() - 1); header.Number.Uint64() != 0; prev, header = header, bc.GetHeaderByNumber(header.Number.Uint64()-1) {
+ if prev.ParentHash != header.Hash() {
+ t.Errorf("parent header hash mismatch: have %x, want %x", prev.ParentHash, header.Hash())
+ }
+ }
+ }
+ // Make sure the chain total difficulty is the correct one
+ want := new(big.Int).Add(genesis.Difficulty(), big.NewInt(td))
+ if full {
+ if have := bc.GetTd(bc.CurrentBlock().Hash()); have.Cmp(want) != 0 {
+ t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
+ }
+ } else {
+ if have := bc.GetTd(bc.CurrentHeader().Hash()); have.Cmp(want) != 0 {
+ t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
}
}
}
-func TestBadHashes(t *testing.T) {
+// Tests that the insertion functions detect banned hashes.
+func TestBadHeaderHashes(t *testing.T) { testBadHashes(t, false) }
+func TestBadBlockHashes(t *testing.T) { testBadHashes(t, true) }
+
+func testBadHashes(t *testing.T, full bool) {
+ // Create a pristine block chain
db, _ := ethdb.NewMemDatabase()
- genesis, err := WriteTestNetGenesisBlock(db, 0)
- if err != nil {
- t.Error(err)
- t.FailNow()
- }
+ genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
- chain := makeChainWithDiff(genesis, []int{1, 2, 4}, 10)
- BadHashes[chain[2].Header().Hash()] = true
-
- _, err = bc.InsertChain(chain)
+ // Create a chain, ban a hash and try to import
+ var err error
+ if full {
+ blocks := makeBlockChainWithDiff(genesis, []int{1, 2, 4}, 10)
+ BadHashes[blocks[2].Header().Hash()] = true
+ _, err = bc.InsertChain(blocks)
+ } else {
+ headers := makeHeaderChainWithDiff(genesis, []int{1, 2, 4}, 10)
+ BadHashes[headers[2].Hash()] = true
+ _, err = bc.InsertHeaderChain(headers, 1)
+ }
if !IsBadHashError(err) {
t.Errorf("error mismatch: want: BadHashError, have: %v", err)
}
}
-func TestReorgBadHashes(t *testing.T) {
+// Tests that bad hashes are detected on boot, and the chan rolled back to a
+// good state prior to the bad hash.
+func TestReorgBadHeaderHashes(t *testing.T) { testReorgBadHashes(t, false) }
+func TestReorgBadBlockHashes(t *testing.T) { testReorgBadHashes(t, true) }
+
+func testReorgBadHashes(t *testing.T, full bool) {
+ // Create a pristine block chain
db, _ := ethdb.NewMemDatabase()
- genesis, err := WriteTestNetGenesisBlock(db, 0)
- if err != nil {
- t.Error(err)
- t.FailNow()
- }
+ genesis, _ := WriteTestNetGenesisBlock(db, 0)
bc := chm(genesis, db)
- chain := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 11)
- bc.InsertChain(chain)
+ // Create a chain, import and ban aferwards
+ headers := makeHeaderChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
+ blocks := makeBlockChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
- if chain[3].Header().Hash() != bc.LastBlockHash() {
- t.Errorf("last block hash mismatch: want: %x, have: %x", chain[3].Header().Hash(), bc.LastBlockHash())
+ if full {
+ if _, err := bc.InsertChain(blocks); err != nil {
+ t.Fatalf("failed to import blocks: %v", err)
+ }
+ if bc.CurrentBlock().Hash() != blocks[3].Hash() {
+ t.Errorf("last block hash mismatch: have: %x, want %x", bc.CurrentBlock().Hash(), blocks[3].Header().Hash())
+ }
+ BadHashes[blocks[3].Header().Hash()] = true
+ defer func() { delete(BadHashes, blocks[3].Header().Hash()) }()
+ } else {
+ if _, err := bc.InsertHeaderChain(headers, 1); err != nil {
+ t.Fatalf("failed to import headers: %v", err)
+ }
+ if bc.CurrentHeader().Hash() != headers[3].Hash() {
+ t.Errorf("last header hash mismatch: have: %x, want %x", bc.CurrentHeader().Hash(), headers[3].Hash())
+ }
+ BadHashes[headers[3].Hash()] = true
+ defer func() { delete(BadHashes, headers[3].Hash()) }()
}
-
- // NewChainManager should check BadHashes when loading it db
- BadHashes[chain[3].Header().Hash()] = true
-
- var eventMux event.TypeMux
- ncm, err := NewBlockChain(db, FakePow{}, &eventMux)
+ // Create a new chain manager and check it rolled back the state
+ ncm, err := NewBlockChain(db, FakePow{}, new(event.TypeMux))
if err != nil {
- t.Errorf("NewChainManager err: %s", err)
+ t.Fatalf("failed to create new chain manager: %v", err)
}
-
- // check it set head to (valid) parent of bad hash block
- if chain[2].Header().Hash() != ncm.LastBlockHash() {
- t.Errorf("last block hash mismatch: want: %x, have: %x", chain[2].Header().Hash(), ncm.LastBlockHash())
- }
-
- if chain[2].Header().GasLimit.Cmp(ncm.GasLimit()) != 0 {
- t.Errorf("current block gasLimit mismatch: want: %x, have: %x", chain[2].Header().GasLimit, ncm.GasLimit())
- }
-}
-
-func TestReorgShortest(t *testing.T) {
- db, _ := ethdb.NewMemDatabase()
- genesis, err := WriteTestNetGenesisBlock(db, 0)
- if err != nil {
- t.Error(err)
- t.FailNow()
- }
- bc := chm(genesis, db)
-
- chain1 := makeChainWithDiff(genesis, []int{1, 2, 3, 4}, 10)
- chain2 := makeChainWithDiff(genesis, []int{1, 10}, 11)
-
- bc.InsertChain(chain1)
- bc.InsertChain(chain2)
-
- prev := bc.CurrentBlock()
- for block := bc.GetBlockByNumber(bc.CurrentBlock().NumberU64() - 1); block.NumberU64() != 0; prev, block = block, bc.GetBlockByNumber(block.NumberU64()-1) {
- if prev.ParentHash() != block.Hash() {
- t.Errorf("parent hash mismatch %x - %x", prev.ParentHash(), block.Hash())
+ if full {
+ if ncm.CurrentBlock().Hash() != blocks[2].Header().Hash() {
+ t.Errorf("last block hash mismatch: have: %x, want %x", ncm.CurrentBlock().Hash(), blocks[2].Header().Hash())
+ }
+ if blocks[2].Header().GasLimit.Cmp(ncm.GasLimit()) != 0 {
+ t.Errorf("last block gasLimit mismatch: have: %x, want %x", ncm.GasLimit(), blocks[2].Header().GasLimit)
+ }
+ } else {
+ if ncm.CurrentHeader().Hash() != headers[2].Hash() {
+ t.Errorf("last header hash mismatch: have: %x, want %x", ncm.CurrentHeader().Hash(), headers[2].Hash())
}
}
}
-func TestInsertNonceError(t *testing.T) {
+// Tests chain insertions in the face of one entity containing an invalid nonce.
+func TestHeadersInsertNonceError(t *testing.T) { testInsertNonceError(t, false) }
+func TestBlocksInsertNonceError(t *testing.T) { testInsertNonceError(t, true) }
+
+func testInsertNonceError(t *testing.T, full bool) {
for i := 1; i < 25 && !t.Failed(); i++ {
- db, _ := ethdb.NewMemDatabase()
- genesis, err := WriteTestNetGenesisBlock(db, 0)
+ // Create a pristine chain and database
+ db, processor, err := newCanonical(0, full)
if err != nil {
- t.Error(err)
- t.FailNow()
+ t.Fatalf("failed to create pristine chain: %v", err)
}
- bc := chm(genesis, db)
- bc.processor = NewBlockProcessor(db, bc.pow, bc, bc.eventMux)
- blocks := makeChain(bc.currentBlock, i, db, 0)
+ bc := processor.bc
- fail := rand.Int() % len(blocks)
- failblock := blocks[fail]
- bc.pow = failPow{failblock.NumberU64()}
- n, err := bc.InsertChain(blocks)
+ // Create and insert a chain with a failing nonce
+ var (
+ failAt int
+ failRes int
+ failNum uint64
+ failHash common.Hash
+ )
+ if full {
+ blocks := makeBlockChain(processor.bc.CurrentBlock(), i, db, 0)
+ failAt = rand.Int() % len(blocks)
+ failNum = blocks[failAt].NumberU64()
+ failHash = blocks[failAt].Hash()
+
+ processor.bc.pow = failPow{failNum}
+ processor.Pow = failPow{failNum}
+
+ failRes, err = processor.bc.InsertChain(blocks)
+ } else {
+ headers := makeHeaderChain(processor.bc.CurrentHeader(), i, db, 0)
+
+ failAt = rand.Int() % len(headers)
+ failNum = headers[failAt].Number.Uint64()
+ failHash = headers[failAt].Hash()
+
+ processor.bc.pow = failPow{failNum}
+ processor.Pow = failPow{failNum}
+
+ failRes, err = processor.bc.InsertHeaderChain(headers, 1)
+ }
// Check that the returned error indicates the nonce failure.
- if n != fail {
- t.Errorf("(i=%d) wrong failed block index: got %d, want %d", i, n, fail)
+ if failRes != failAt {
+ t.Errorf("test %d: failure index mismatch: have %d, want %d", i, failRes, failAt)
}
if !IsBlockNonceErr(err) {
- t.Fatalf("(i=%d) got %q, want a nonce error", i, err)
+ t.Fatalf("test %d: error mismatch: have %v, want nonce error", i, err)
}
nerr := err.(*BlockNonceErr)
- if nerr.Number.Cmp(failblock.Number()) != 0 {
- t.Errorf("(i=%d) wrong block number in error, got %v, want %v", i, nerr.Number, failblock.Number())
+ if nerr.Number.Uint64() != failNum {
+ t.Errorf("test %d: number mismatch: have %v, want %v", i, nerr.Number, failNum)
}
- if nerr.Hash != failblock.Hash() {
- t.Errorf("(i=%d) wrong block hash in error, got %v, want %v", i, nerr.Hash, failblock.Hash())
+ if nerr.Hash != failHash {
+ t.Errorf("test %d: hash mismatch: have %x, want %x", i, nerr.Hash[:4], failHash[:4])
}
-
// Check that all no blocks after the failing block have been inserted.
- for _, block := range blocks[fail:] {
- if bc.HasBlock(block.Hash()) {
- t.Errorf("(i=%d) invalid block %d present in chain", i, block.NumberU64())
+ for j := 0; j < i-failAt; j++ {
+ if full {
+ if block := bc.GetBlockByNumber(failNum + uint64(j)); block != nil {
+ t.Errorf("test %d: invalid block in chain: %v", i, block)
+ }
+ } else {
+ if header := bc.GetHeaderByNumber(failNum + uint64(j)); header != nil {
+ t.Errorf("test %d: invalid header in chain: %v", i, header)
+ }
}
}
}
}
+// Tests that fast importing a block chain produces the same chain data as the
+// classical full block processing.
+func TestFastVsFullChains(t *testing.T) {
+ // Configure and generate a sample block chain
+ var (
+ gendb, _ = ethdb.NewMemDatabase()
+ key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ address = crypto.PubkeyToAddress(key.PublicKey)
+ funds = big.NewInt(1000000000)
+ genesis = GenesisBlockForTesting(gendb, address, funds)
+ )
+ blocks, receipts := GenerateChain(genesis, gendb, 1024, func(i int, block *BlockGen) {
+ block.SetCoinbase(common.Address{0x00})
+
+ // If the block number is multiple of 3, send a few bonus transactions to the miner
+ if i%3 == 2 {
+ for j := 0; j < i%4+1; j++ {
+ tx, err := types.NewTransaction(block.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key)
+ if err != nil {
+ panic(err)
+ }
+ block.AddTx(tx)
+ }
+ }
+ // If the block number is a multiple of 5, add a few bonus uncles to the block
+ if i%5 == 5 {
+ block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))})
+ }
+ })
+ // Import the chain as an archive node for the comparison baseline
+ archiveDb, _ := ethdb.NewMemDatabase()
+ WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
+
+ archive, _ := NewBlockChain(archiveDb, FakePow{}, new(event.TypeMux))
+ archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux)))
+
+ if n, err := archive.InsertChain(blocks); err != nil {
+ t.Fatalf("failed to process block %d: %v", n, err)
+ }
+ // Fast import the chain as a non-archive node to test
+ fastDb, _ := ethdb.NewMemDatabase()
+ WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
+ fast, _ := NewBlockChain(fastDb, FakePow{}, new(event.TypeMux))
+ fast.SetProcessor(NewBlockProcessor(fastDb, FakePow{}, fast, new(event.TypeMux)))
+
+ headers := make([]*types.Header, len(blocks))
+ for i, block := range blocks {
+ headers[i] = block.Header()
+ }
+ if n, err := fast.InsertHeaderChain(headers, 1); err != nil {
+ t.Fatalf("failed to insert header %d: %v", n, err)
+ }
+ if n, err := fast.InsertReceiptChain(blocks, receipts); err != nil {
+ t.Fatalf("failed to insert receipt %d: %v", n, err)
+ }
+ // Iterate over all chain data components, and cross reference
+ for i := 0; i < len(blocks); i++ {
+ num, hash := blocks[i].NumberU64(), blocks[i].Hash()
+
+ if ftd, atd := fast.GetTd(hash), archive.GetTd(hash); ftd.Cmp(atd) != 0 {
+ t.Errorf("block #%d [%x]: td mismatch: have %v, want %v", num, hash, ftd, atd)
+ }
+ if fheader, aheader := fast.GetHeader(hash), archive.GetHeader(hash); fheader.Hash() != aheader.Hash() {
+ t.Errorf("block #%d [%x]: header mismatch: have %v, want %v", num, hash, fheader, aheader)
+ }
+ if fblock, ablock := fast.GetBlock(hash), archive.GetBlock(hash); fblock.Hash() != ablock.Hash() {
+ t.Errorf("block #%d [%x]: block mismatch: have %v, want %v", num, hash, fblock, ablock)
+ } else if types.DeriveSha(fblock.Transactions()) != types.DeriveSha(ablock.Transactions()) {
+ t.Errorf("block #%d [%x]: transactions mismatch: have %v, want %v", num, hash, fblock.Transactions(), ablock.Transactions())
+ } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(ablock.Uncles()) {
+ t.Errorf("block #%d [%x]: uncles mismatch: have %v, want %v", num, hash, fblock.Uncles(), ablock.Uncles())
+ }
+ if freceipts, areceipts := GetBlockReceipts(fastDb, hash), GetBlockReceipts(archiveDb, hash); types.DeriveSha(freceipts) != types.DeriveSha(areceipts) {
+ t.Errorf("block #%d [%x]: receipts mismatch: have %v, want %v", num, hash, freceipts, areceipts)
+ }
+ }
+ // Check that the canonical chains are the same between the databases
+ for i := 0; i < len(blocks)+1; i++ {
+ if fhash, ahash := GetCanonicalHash(fastDb, uint64(i)), GetCanonicalHash(archiveDb, uint64(i)); fhash != ahash {
+ t.Errorf("block #%d: canonical hash mismatch: have %v, want %v", i, fhash, ahash)
+ }
+ }
+}
+
+// Tests that various import methods move the chain head pointers to the correct
+// positions.
+func TestLightVsFastVsFullChainHeads(t *testing.T) {
+ // Configure and generate a sample block chain
+ var (
+ gendb, _ = ethdb.NewMemDatabase()
+ key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+ address = crypto.PubkeyToAddress(key.PublicKey)
+ funds = big.NewInt(1000000000)
+ genesis = GenesisBlockForTesting(gendb, address, funds)
+ )
+ height := uint64(1024)
+ blocks, receipts := GenerateChain(genesis, gendb, int(height), nil)
+
+ // Configure a subchain to roll back
+ remove := []common.Hash{}
+ for _, block := range blocks[height/2:] {
+ remove = append(remove, block.Hash())
+ }
+ // Create a small assertion method to check the three heads
+ assert := func(t *testing.T, kind string, chain *BlockChain, header uint64, fast uint64, block uint64) {
+ if num := chain.CurrentBlock().NumberU64(); num != block {
+ t.Errorf("%s head block mismatch: have #%v, want #%v", kind, num, block)
+ }
+ if num := chain.CurrentFastBlock().NumberU64(); num != fast {
+ t.Errorf("%s head fast-block mismatch: have #%v, want #%v", kind, num, fast)
+ }
+ if num := chain.CurrentHeader().Number.Uint64(); num != header {
+ t.Errorf("%s head header mismatch: have #%v, want #%v", kind, num, header)
+ }
+ }
+ // Import the chain as an archive node and ensure all pointers are updated
+ archiveDb, _ := ethdb.NewMemDatabase()
+ WriteGenesisBlockForTesting(archiveDb, GenesisAccount{address, funds})
+
+ archive, _ := NewBlockChain(archiveDb, FakePow{}, new(event.TypeMux))
+ archive.SetProcessor(NewBlockProcessor(archiveDb, FakePow{}, archive, new(event.TypeMux)))
+
+ if n, err := archive.InsertChain(blocks); err != nil {
+ t.Fatalf("failed to process block %d: %v", n, err)
+ }
+ assert(t, "archive", archive, height, height, height)
+ archive.Rollback(remove)
+ assert(t, "archive", archive, height/2, height/2, height/2)
+
+ // Import the chain as a non-archive node and ensure all pointers are updated
+ fastDb, _ := ethdb.NewMemDatabase()
+ WriteGenesisBlockForTesting(fastDb, GenesisAccount{address, funds})
+ fast, _ := NewBlockChain(fastDb, FakePow{}, new(event.TypeMux))
+ fast.SetProcessor(NewBlockProcessor(fastDb, FakePow{}, fast, new(event.TypeMux)))
+
+ headers := make([]*types.Header, len(blocks))
+ for i, block := range blocks {
+ headers[i] = block.Header()
+ }
+ if n, err := fast.InsertHeaderChain(headers, 1); err != nil {
+ t.Fatalf("failed to insert header %d: %v", n, err)
+ }
+ if n, err := fast.InsertReceiptChain(blocks, receipts); err != nil {
+ t.Fatalf("failed to insert receipt %d: %v", n, err)
+ }
+ assert(t, "fast", fast, height, height, 0)
+ fast.Rollback(remove)
+ assert(t, "fast", fast, height/2, height/2, 0)
+
+ // Import the chain as a light node and ensure all pointers are updated
+ lightDb, _ := ethdb.NewMemDatabase()
+ WriteGenesisBlockForTesting(lightDb, GenesisAccount{address, funds})
+ light, _ := NewBlockChain(lightDb, FakePow{}, new(event.TypeMux))
+ light.SetProcessor(NewBlockProcessor(lightDb, FakePow{}, light, new(event.TypeMux)))
+
+ if n, err := light.InsertHeaderChain(headers, 1); err != nil {
+ t.Fatalf("failed to insert header %d: %v", n, err)
+ }
+ assert(t, "light", light, height, 0, 0)
+ light.Rollback(remove)
+ assert(t, "light", light, height/2, 0, 0)
+}
+
// Tests that chain reorganizations handle transaction removals and reinsertions.
func TestChainTxReorgs(t *testing.T) {
params.MinGasLimit = big.NewInt(125000) // Minimum the gas limit may ever be.
@@ -587,7 +876,7 @@ func TestChainTxReorgs(t *testing.T) {
// - futureAdd: transaction added after the reorg has already finished
var pastAdd, freshAdd, futureAdd *types.Transaction
- chain := GenerateChain(genesis, db, 3, func(i int, gen *BlockGen) {
+ chain, _ := GenerateChain(genesis, db, 3, func(i int, gen *BlockGen) {
switch i {
case 0:
pastDrop, _ = types.NewTransaction(gen.TxNonce(addr2), addr2, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key2)
@@ -613,7 +902,7 @@ func TestChainTxReorgs(t *testing.T) {
}
// overwrite the old chain
- chain = GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
+ chain, _ = GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
pastAdd, _ = types.NewTransaction(gen.TxNonce(addr3), addr3, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(key3)
diff --git a/core/chain_makers.go b/core/chain_makers.go
index e20a05c7d..56e37a0fc 100644
--- a/core/chain_makers.go
+++ b/core/chain_makers.go
@@ -98,7 +98,7 @@ func (b *BlockGen) AddTx(tx *types.Transaction) {
b.header.GasUsed.Add(b.header.GasUsed, gas)
receipt := types.NewReceipt(root.Bytes(), b.header.GasUsed)
logs := b.statedb.GetLogs(tx.Hash())
- receipt.SetLogs(logs)
+ receipt.Logs = logs
receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
b.txs = append(b.txs, tx)
b.receipts = append(b.receipts, receipt)
@@ -163,13 +163,13 @@ func (b *BlockGen) OffsetTime(seconds int64) {
// Blocks created by GenerateChain do not contain valid proof of work
// values. Inserting them into BlockChain requires use of FakePow or
// a similar non-validating proof of work implementation.
-func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) []*types.Block {
+func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int, *BlockGen)) ([]*types.Block, []types.Receipts) {
statedb, err := state.New(parent.Root(), db)
if err != nil {
panic(err)
}
- blocks := make(types.Blocks, n)
- genblock := func(i int, h *types.Header) *types.Block {
+ blocks, receipts := make(types.Blocks, n), make([]types.Receipts, n)
+ genblock := func(i int, h *types.Header) (*types.Block, types.Receipts) {
b := &BlockGen{parent: parent, i: i, chain: blocks, header: h, statedb: statedb}
if gen != nil {
gen(i, b)
@@ -180,15 +180,16 @@ func GenerateChain(parent *types.Block, db ethdb.Database, n int, gen func(int,
panic(fmt.Sprintf("state write error: %v", err))
}
h.Root = root
- return types.NewBlock(h, b.txs, b.uncles, b.receipts)
+ return types.NewBlock(h, b.txs, b.uncles, b.receipts), b.receipts
}
for i := 0; i < n; i++ {
header := makeHeader(parent, statedb)
- block := genblock(i, header)
+ block, receipt := genblock(i, header)
blocks[i] = block
+ receipts[i] = receipt
parent = block
}
- return blocks
+ return blocks, receipts
}
func makeHeader(parent *types.Block, state *state.StateDB) *types.Header {
@@ -210,26 +211,51 @@ func makeHeader(parent *types.Block, state *state.StateDB) *types.Header {
}
}
-// newCanonical creates a new deterministic canonical chain by running
-// InsertChain on the result of makeChain.
-func newCanonical(n int, db ethdb.Database) (*BlockProcessor, error) {
+// newCanonical creates a chain database, and injects a deterministic canonical
+// chain. Depending on the full flag, if creates either a full block chain or a
+// header only chain.
+func newCanonical(n int, full bool) (ethdb.Database, *BlockProcessor, error) {
+ // Create te new chain database
+ db, _ := ethdb.NewMemDatabase()
evmux := &event.TypeMux{}
- WriteTestNetGenesisBlock(db, 0)
- chainman, _ := NewBlockChain(db, FakePow{}, evmux)
- bman := NewBlockProcessor(db, FakePow{}, chainman, evmux)
- bman.bc.SetProcessor(bman)
- parent := bman.bc.CurrentBlock()
+ // Initialize a fresh chain with only a genesis block
+ genesis, _ := WriteTestNetGenesisBlock(db, 0)
+
+ blockchain, _ := NewBlockChain(db, FakePow{}, evmux)
+ processor := NewBlockProcessor(db, FakePow{}, blockchain, evmux)
+ processor.bc.SetProcessor(processor)
+
+ // Create and inject the requested chain
if n == 0 {
- return bman, nil
+ return db, processor, nil
}
- lchain := makeChain(parent, n, db, canonicalSeed)
- _, err := bman.bc.InsertChain(lchain)
- return bman, err
+ if full {
+ // Full block-chain requested
+ blocks := makeBlockChain(genesis, n, db, canonicalSeed)
+ _, err := blockchain.InsertChain(blocks)
+ return db, processor, err
+ }
+ // Header-only chain requested
+ headers := makeHeaderChain(genesis.Header(), n, db, canonicalSeed)
+ _, err := blockchain.InsertHeaderChain(headers, 1)
+ return db, processor, err
}
-func makeChain(parent *types.Block, n int, db ethdb.Database, seed int) []*types.Block {
- return GenerateChain(parent, db, n, func(i int, b *BlockGen) {
+// makeHeaderChain creates a deterministic chain of headers rooted at parent.
+func makeHeaderChain(parent *types.Header, n int, db ethdb.Database, seed int) []*types.Header {
+ blocks := makeBlockChain(types.NewBlockWithHeader(parent), n, db, seed)
+ headers := make([]*types.Header, len(blocks))
+ for i, block := range blocks {
+ headers[i] = block.Header()
+ }
+ return headers
+}
+
+// makeBlockChain creates a deterministic chain of blocks rooted at parent.
+func makeBlockChain(parent *types.Block, n int, db ethdb.Database, seed int) []*types.Block {
+ blocks, _ := GenerateChain(parent, db, n, func(i int, b *BlockGen) {
b.SetCoinbase(common.Address{0: byte(seed), 19: byte(i)})
})
+ return blocks
}
diff --git a/core/chain_makers_test.go b/core/chain_makers_test.go
index 63825c261..7f47cf288 100644
--- a/core/chain_makers_test.go
+++ b/core/chain_makers_test.go
@@ -47,7 +47,7 @@ func ExampleGenerateChain() {
// This call generates a chain of 5 blocks. The function runs for
// each block and adds different features to gen based on the
// block index.
- chain := GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
+ chain, _ := GenerateChain(genesis, db, 5, func(i int, gen *BlockGen) {
switch i {
case 0:
// In block 1, addr1 sends addr2 some ether.
diff --git a/core/chain_pow_test.go b/core/chain_pow_test.go
index 5aa8ed8a0..d2b0bd144 100644
--- a/core/chain_pow_test.go
+++ b/core/chain_pow_test.go
@@ -60,7 +60,7 @@ func TestPowVerification(t *testing.T) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
- blocks = GenerateChain(genesis, testdb, 8, nil)
+ blocks, _ = GenerateChain(genesis, testdb, 8, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
@@ -115,7 +115,7 @@ func testPowConcurrentVerification(t *testing.T, threads int) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
- blocks = GenerateChain(genesis, testdb, 8, nil)
+ blocks, _ = GenerateChain(genesis, testdb, 8, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
@@ -186,7 +186,7 @@ func testPowConcurrentAbortion(t *testing.T, threads int) {
var (
testdb, _ = ethdb.NewMemDatabase()
genesis = GenesisBlockForTesting(testdb, common.Address{}, new(big.Int))
- blocks = GenerateChain(genesis, testdb, 1024, nil)
+ blocks, _ = GenerateChain(genesis, testdb, 1024, nil)
)
headers := make([]*types.Header, len(blocks))
for i, block := range blocks {
diff --git a/core/chain_util.go b/core/chain_util.go
index 42b6a5be2..ddff381a1 100644
--- a/core/chain_util.go
+++ b/core/chain_util.go
@@ -34,6 +34,7 @@ import (
var (
headHeaderKey = []byte("LastHeader")
headBlockKey = []byte("LastBlock")
+ headFastKey = []byte("LastFast")
blockPrefix = []byte("block-")
blockNumPrefix = []byte("block-num-")
@@ -129,7 +130,7 @@ func GetCanonicalHash(db ethdb.Database, number uint64) common.Hash {
// header. The difference between this and GetHeadBlockHash is that whereas the
// last block hash is only updated upon a full block import, the last header
// hash is updated already at header import, allowing head tracking for the
-// fast synchronization mechanism.
+// light synchronization mechanism.
func GetHeadHeaderHash(db ethdb.Database) common.Hash {
data, _ := db.Get(headHeaderKey)
if len(data) == 0 {
@@ -147,6 +148,18 @@ func GetHeadBlockHash(db ethdb.Database) common.Hash {
return common.BytesToHash(data)
}
+// GetHeadFastBlockHash retrieves the hash of the current canonical head block during
+// fast synchronization. The difference between this and GetHeadBlockHash is that
+// whereas the last block hash is only updated upon a full block import, the last
+// fast hash is updated when importing pre-processed blocks.
+func GetHeadFastBlockHash(db ethdb.Database) common.Hash {
+ data, _ := db.Get(headFastKey)
+ if len(data) == 0 {
+ return common.Hash{}
+ }
+ return common.BytesToHash(data)
+}
+
// GetHeaderRLP retrieves a block header in its raw RLP database encoding, or nil
// if the header's not found.
func GetHeaderRLP(db ethdb.Database, hash common.Hash) rlp.RawValue {
@@ -249,6 +262,15 @@ func WriteHeadBlockHash(db ethdb.Database, hash common.Hash) error {
return nil
}
+// WriteHeadFastBlockHash stores the fast head block's hash.
+func WriteHeadFastBlockHash(db ethdb.Database, hash common.Hash) error {
+ if err := db.Put(headFastKey, hash.Bytes()); err != nil {
+ glog.Fatalf("failed to store last fast block's hash into database: %v", err)
+ return err
+ }
+ return nil
+}
+
// WriteHeader serializes a block header into the database.
func WriteHeader(db ethdb.Database, header *types.Header) error {
data, err := rlp.EncodeToBytes(header)
@@ -372,7 +394,7 @@ func WriteMipmapBloom(db ethdb.Database, number uint64, receipts types.Receipts)
bloomDat, _ := db.Get(key)
bloom := types.BytesToBloom(bloomDat)
for _, receipt := range receipts {
- for _, log := range receipt.Logs() {
+ for _, log := range receipt.Logs {
bloom.Add(log.Address.Big())
}
}
diff --git a/core/chain_util_test.go b/core/chain_util_test.go
index 62b73a064..0bbcbbe53 100644
--- a/core/chain_util_test.go
+++ b/core/chain_util_test.go
@@ -163,7 +163,12 @@ func TestBlockStorage(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
// Create a test block to move around the database and make sure it's really new
- block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")})
+ block := types.NewBlockWithHeader(&types.Header{
+ Extra: []byte("test block"),
+ UncleHash: types.EmptyUncleHash,
+ TxHash: types.EmptyRootHash,
+ ReceiptHash: types.EmptyRootHash,
+ })
if entry := GetBlock(db, block.Hash()); entry != nil {
t.Fatalf("Non existent block returned: %v", entry)
}
@@ -208,8 +213,12 @@ func TestBlockStorage(t *testing.T) {
// Tests that partial block contents don't get reassembled into full blocks.
func TestPartialBlockStorage(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
- block := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block")})
-
+ block := types.NewBlockWithHeader(&types.Header{
+ Extra: []byte("test block"),
+ UncleHash: types.EmptyUncleHash,
+ TxHash: types.EmptyRootHash,
+ ReceiptHash: types.EmptyRootHash,
+ })
// Store a header and check that it's not recognized as a block
if err := WriteHeader(db, block.Header()); err != nil {
t.Fatalf("Failed to write header into database: %v", err)
@@ -298,6 +307,7 @@ func TestHeadStorage(t *testing.T) {
blockHead := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block header")})
blockFull := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block full")})
+ blockFast := types.NewBlockWithHeader(&types.Header{Extra: []byte("test block fast")})
// Check that no head entries are in a pristine database
if entry := GetHeadHeaderHash(db); entry != (common.Hash{}) {
@@ -306,6 +316,9 @@ func TestHeadStorage(t *testing.T) {
if entry := GetHeadBlockHash(db); entry != (common.Hash{}) {
t.Fatalf("Non head block entry returned: %v", entry)
}
+ if entry := GetHeadFastBlockHash(db); entry != (common.Hash{}) {
+ t.Fatalf("Non fast head block entry returned: %v", entry)
+ }
// Assign separate entries for the head header and block
if err := WriteHeadHeaderHash(db, blockHead.Hash()); err != nil {
t.Fatalf("Failed to write head header hash: %v", err)
@@ -313,6 +326,9 @@ func TestHeadStorage(t *testing.T) {
if err := WriteHeadBlockHash(db, blockFull.Hash()); err != nil {
t.Fatalf("Failed to write head block hash: %v", err)
}
+ if err := WriteHeadFastBlockHash(db, blockFast.Hash()); err != nil {
+ t.Fatalf("Failed to write fast head block hash: %v", err)
+ }
// Check that both heads are present, and different (i.e. two heads maintained)
if entry := GetHeadHeaderHash(db); entry != blockHead.Hash() {
t.Fatalf("Head header hash mismatch: have %v, want %v", entry, blockHead.Hash())
@@ -320,21 +336,24 @@ func TestHeadStorage(t *testing.T) {
if entry := GetHeadBlockHash(db); entry != blockFull.Hash() {
t.Fatalf("Head block hash mismatch: have %v, want %v", entry, blockFull.Hash())
}
+ if entry := GetHeadFastBlockHash(db); entry != blockFast.Hash() {
+ t.Fatalf("Fast head block hash mismatch: have %v, want %v", entry, blockFast.Hash())
+ }
}
func TestMipmapBloom(t *testing.T) {
db, _ := ethdb.NewMemDatabase()
receipt1 := new(types.Receipt)
- receipt1.SetLogs(vm.Logs{
+ receipt1.Logs = vm.Logs{
&vm.Log{Address: common.BytesToAddress([]byte("test"))},
&vm.Log{Address: common.BytesToAddress([]byte("address"))},
- })
+ }
receipt2 := new(types.Receipt)
- receipt2.SetLogs(vm.Logs{
+ receipt2.Logs = vm.Logs{
&vm.Log{Address: common.BytesToAddress([]byte("test"))},
&vm.Log{Address: common.BytesToAddress([]byte("address1"))},
- })
+ }
WriteMipmapBloom(db, 1, types.Receipts{receipt1})
WriteMipmapBloom(db, 2, types.Receipts{receipt2})
@@ -349,15 +368,15 @@ func TestMipmapBloom(t *testing.T) {
// reset
db, _ = ethdb.NewMemDatabase()
receipt := new(types.Receipt)
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{Address: common.BytesToAddress([]byte("test"))},
- })
+ }
WriteMipmapBloom(db, 999, types.Receipts{receipt1})
receipt = new(types.Receipt)
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{Address: common.BytesToAddress([]byte("test 1"))},
- })
+ }
WriteMipmapBloom(db, 1000, types.Receipts{receipt})
bloom := GetMipmapBloom(db, 1000, 1000)
@@ -384,22 +403,22 @@ func TestMipmapChain(t *testing.T) {
defer db.Close()
genesis := WriteGenesisBlockForTesting(db, GenesisAccount{addr, big.NewInt(1000000)})
- chain := GenerateChain(genesis, db, 1010, func(i int, gen *BlockGen) {
+ chain, receipts := GenerateChain(genesis, db, 1010, func(i int, gen *BlockGen) {
var receipts types.Receipts
switch i {
case 1:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{
Address: addr,
Topics: []common.Hash{hash1},
},
- })
+ }
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 1000:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{&vm.Log{Address: addr2}})
+ receipt.Logs = vm.Logs{&vm.Log{Address: addr2}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
@@ -412,7 +431,7 @@ func TestMipmapChain(t *testing.T) {
}
WriteMipmapBloom(db, uint64(i+1), receipts)
})
- for _, block := range chain {
+ for i, block := range chain {
WriteBlock(db, block)
if err := WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
@@ -420,7 +439,7 @@ func TestMipmapChain(t *testing.T) {
if err := WriteHeadBlockHash(db, block.Hash()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
- if err := PutBlockReceipts(db, block, block.Receipts()); err != nil {
+ if err := PutBlockReceipts(db, block.Hash(), receipts[i]); err != nil {
t.Fatal("error writing block receipts:", err)
}
}
diff --git a/core/error.go b/core/error.go
index 6498194cd..0ba506f46 100644
--- a/core/error.go
+++ b/core/error.go
@@ -111,7 +111,7 @@ type BlockNonceErr struct {
}
func (err *BlockNonceErr) Error() string {
- return fmt.Sprintf("block %d (%v) nonce is invalid (got %d)", err.Number, err.Hash, err.Nonce)
+ return fmt.Sprintf("nonce for #%d [%x…] is invalid (got %d)", err.Number, err.Hash, err.Nonce)
}
// IsBlockNonceErr returns true for invalid block nonce errors.
diff --git a/core/genesis.go b/core/genesis.go
index 16c1598c2..dac5de92f 100644
--- a/core/genesis.go
+++ b/core/genesis.go
@@ -103,7 +103,7 @@ func WriteGenesisBlock(chainDb ethdb.Database, reader io.Reader) (*types.Block,
if err := WriteBlock(chainDb, block); err != nil {
return nil, err
}
- if err := PutBlockReceipts(chainDb, block, nil); err != nil {
+ if err := PutBlockReceipts(chainDb, block.Hash(), nil); err != nil {
return nil, err
}
if err := WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()); err != nil {
diff --git a/core/state/sync.go b/core/state/sync.go
new file mode 100644
index 000000000..ef2b4b84c
--- /dev/null
+++ b/core/state/sync.go
@@ -0,0 +1,70 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package state
+
+import (
+ "bytes"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/rlp"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// StateSync is the main state synchronisation scheduler, which provides yet the
+// unknown state hashes to retrieve, accepts node data associated with said hashes
+// and reconstructs the state database step by step until all is done.
+type StateSync trie.TrieSync
+
+// NewStateSync create a new state trie download scheduler.
+func NewStateSync(root common.Hash, database ethdb.Database) *StateSync {
+ var syncer *trie.TrieSync
+
+ callback := func(leaf []byte, parent common.Hash) error {
+ var obj struct {
+ Nonce uint64
+ Balance *big.Int
+ Root common.Hash
+ CodeHash []byte
+ }
+ if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil {
+ return err
+ }
+ syncer.AddSubTrie(obj.Root, 64, parent, nil)
+ syncer.AddRawEntry(common.BytesToHash(obj.CodeHash), 64, parent)
+
+ return nil
+ }
+ syncer = trie.NewTrieSync(root, database, callback)
+ return (*StateSync)(syncer)
+}
+
+// Missing retrieves the known missing nodes from the state trie for retrieval.
+func (s *StateSync) Missing(max int) []common.Hash {
+ return (*trie.TrieSync)(s).Missing(max)
+}
+
+// Process injects a batch of retrieved trie nodes data.
+func (s *StateSync) Process(list []trie.SyncResult) (int, error) {
+ return (*trie.TrieSync)(s).Process(list)
+}
+
+// Pending returns the number of state entries currently pending for download.
+func (s *StateSync) Pending() int {
+ return (*trie.TrieSync)(s).Pending()
+}
diff --git a/core/state/sync_test.go b/core/state/sync_test.go
new file mode 100644
index 000000000..0dab372ba
--- /dev/null
+++ b/core/state/sync_test.go
@@ -0,0 +1,238 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package state
+
+import (
+ "bytes"
+ "math/big"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "github.com/ethereum/go-ethereum/trie"
+)
+
+// testAccount is the data associated with an account used by the state tests.
+type testAccount struct {
+ address common.Address
+ balance *big.Int
+ nonce uint64
+ code []byte
+}
+
+// makeTestState create a sample test state to test node-wise reconstruction.
+func makeTestState() (ethdb.Database, common.Hash, []*testAccount) {
+ // Create an empty state
+ db, _ := ethdb.NewMemDatabase()
+ state, _ := New(common.Hash{}, db)
+
+ // Fill it with some arbitrary data
+ accounts := []*testAccount{}
+ for i := byte(0); i < 255; i++ {
+ obj := state.GetOrNewStateObject(common.BytesToAddress([]byte{i}))
+ acc := &testAccount{address: common.BytesToAddress([]byte{i})}
+
+ obj.AddBalance(big.NewInt(int64(11 * i)))
+ acc.balance = big.NewInt(int64(11 * i))
+
+ obj.SetNonce(uint64(42 * i))
+ acc.nonce = uint64(42 * i)
+
+ if i%3 == 0 {
+ obj.SetCode([]byte{i, i, i, i, i})
+ acc.code = []byte{i, i, i, i, i}
+ }
+ state.UpdateStateObject(obj)
+ accounts = append(accounts, acc)
+ }
+ root, _ := state.Commit()
+
+ // Return the generated state
+ return db, root, accounts
+}
+
+// checkStateAccounts cross references a reconstructed state with an expected
+// account array.
+func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accounts []*testAccount) {
+ state, _ := New(root, db)
+ for i, acc := range accounts {
+
+ if balance := state.GetBalance(acc.address); balance.Cmp(acc.balance) != 0 {
+ t.Errorf("account %d: balance mismatch: have %v, want %v", i, balance, acc.balance)
+ }
+ if nonce := state.GetNonce(acc.address); nonce != acc.nonce {
+ t.Errorf("account %d: nonce mismatch: have %v, want %v", i, nonce, acc.nonce)
+ }
+ if code := state.GetCode(acc.address); bytes.Compare(code, acc.code) != 0 {
+ t.Errorf("account %d: code mismatch: have %x, want %x", i, code, acc.code)
+ }
+ }
+}
+
+// Tests that an empty state is not scheduled for syncing.
+func TestEmptyStateSync(t *testing.T) {
+ empty := common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+ db, _ := ethdb.NewMemDatabase()
+ if req := NewStateSync(empty, db).Missing(1); len(req) != 0 {
+ t.Errorf("content requested for empty state: %v", req)
+ }
+}
+
+// Tests that given a root hash, a state can sync iteratively on a single thread,
+// requesting retrieval tasks and returning all of them in one go.
+func TestIterativeStateSyncIndividual(t *testing.T) { testIterativeStateSync(t, 1) }
+func TestIterativeStateSyncBatched(t *testing.T) { testIterativeStateSync(t, 100) }
+
+func testIterativeStateSync(t *testing.T, batch int) {
+ // Create a random state to copy
+ srcDb, srcRoot, srcAccounts := makeTestState()
+
+ // Create a destination state and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewStateSync(srcRoot, dstDb)
+
+ queue := append([]common.Hash{}, sched.Missing(batch)...)
+ for len(queue) > 0 {
+ results := make([]trie.SyncResult, len(queue))
+ for i, hash := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = trie.SyncResult{hash, data}
+ }
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = append(queue[:0], sched.Missing(batch)...)
+ }
+ // Cross check that the two states are in sync
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+}
+
+// Tests that the trie scheduler can correctly reconstruct the state even if only
+// partial results are returned, and the others sent only later.
+func TestIterativeDelayedStateSync(t *testing.T) {
+ // Create a random state to copy
+ srcDb, srcRoot, srcAccounts := makeTestState()
+
+ // Create a destination state and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewStateSync(srcRoot, dstDb)
+
+ queue := append([]common.Hash{}, sched.Missing(0)...)
+ for len(queue) > 0 {
+ // Sync only half of the scheduled nodes
+ results := make([]trie.SyncResult, len(queue)/2+1)
+ for i, hash := range queue[:len(results)] {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = trie.SyncResult{hash, data}
+ }
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = append(queue[len(results):], sched.Missing(0)...)
+ }
+ // Cross check that the two states are in sync
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+}
+
+// Tests that given a root hash, a trie can sync iteratively on a single thread,
+// requesting retrieval tasks and returning all of them in one go, however in a
+// random order.
+func TestIterativeRandomStateSyncIndividual(t *testing.T) { testIterativeRandomStateSync(t, 1) }
+func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomStateSync(t, 100) }
+
+func testIterativeRandomStateSync(t *testing.T, batch int) {
+ // Create a random state to copy
+ srcDb, srcRoot, srcAccounts := makeTestState()
+
+ // Create a destination state and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewStateSync(srcRoot, dstDb)
+
+ queue := make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(batch) {
+ queue[hash] = struct{}{}
+ }
+ for len(queue) > 0 {
+ // Fetch all the queued nodes in a random order
+ results := make([]trie.SyncResult, 0, len(queue))
+ for hash, _ := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results = append(results, trie.SyncResult{hash, data})
+ }
+ // Feed the retrieved results back and queue new tasks
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(batch) {
+ queue[hash] = struct{}{}
+ }
+ }
+ // Cross check that the two states are in sync
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+}
+
+// Tests that the trie scheduler can correctly reconstruct the state even if only
+// partial results are returned (Even those randomly), others sent only later.
+func TestIterativeRandomDelayedStateSync(t *testing.T) {
+ // Create a random state to copy
+ srcDb, srcRoot, srcAccounts := makeTestState()
+
+ // Create a destination state and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewStateSync(srcRoot, dstDb)
+
+ queue := make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(0) {
+ queue[hash] = struct{}{}
+ }
+ for len(queue) > 0 {
+ // Sync only half of the scheduled nodes, even those in random order
+ results := make([]trie.SyncResult, 0, len(queue)/2+1)
+ for hash, _ := range queue {
+ delete(queue, hash)
+
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results = append(results, trie.SyncResult{hash, data})
+
+ if len(results) >= cap(results) {
+ break
+ }
+ }
+ // Feed the retrieved results back and queue new tasks
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ for _, hash := range sched.Missing(0) {
+ queue[hash] = struct{}{}
+ }
+ }
+ // Cross check that the two states are in sync
+ checkStateAccounts(t, dstDb, srcRoot, srcAccounts)
+}
diff --git a/core/transaction_util.go b/core/transaction_util.go
index d55ed14da..1a3681341 100644
--- a/core/transaction_util.go
+++ b/core/transaction_util.go
@@ -140,11 +140,14 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
if len(data) == 0 {
return nil
}
-
- var receipts types.Receipts
- err := rlp.DecodeBytes(data, &receipts)
- if err != nil {
- glog.V(logger.Core).Infoln("GetReceiptse err", err)
+ rs := []*types.ReceiptForStorage{}
+ if err := rlp.DecodeBytes(data, &rs); err != nil {
+ glog.V(logger.Error).Infof("invalid receipt array RLP for hash %x: %v", hash, err)
+ return nil
+ }
+ receipts := make(types.Receipts, len(rs))
+ for i, receipt := range rs {
+ receipts[i] = (*types.Receipt)(receipt)
}
return receipts
}
@@ -152,7 +155,7 @@ func GetBlockReceipts(db ethdb.Database, hash common.Hash) types.Receipts {
// PutBlockReceipts stores the block's transactions associated receipts
// and stores them by block hash in a single slice. This is required for
// forks and chain reorgs
-func PutBlockReceipts(db ethdb.Database, block *types.Block, receipts types.Receipts) error {
+func PutBlockReceipts(db ethdb.Database, hash common.Hash, receipts types.Receipts) error {
rs := make([]*types.ReceiptForStorage, len(receipts))
for i, receipt := range receipts {
rs[i] = (*types.ReceiptForStorage)(receipt)
@@ -161,12 +164,9 @@ func PutBlockReceipts(db ethdb.Database, block *types.Block, receipts types.Rece
if err != nil {
return err
}
-
- hash := block.Hash()
err = db.Put(append(blockReceiptsPre, hash[:]...), bytes)
if err != nil {
return err
}
-
return nil
}
diff --git a/core/types/block.go b/core/types/block.go
index 7a84045a6..1d1cfa515 100644
--- a/core/types/block.go
+++ b/core/types/block.go
@@ -128,7 +128,6 @@ type Block struct {
header *Header
uncles []*Header
transactions Transactions
- receipts Receipts
// caches
hash atomic.Value
@@ -172,8 +171,8 @@ type storageblock struct {
}
var (
- emptyRootHash = DeriveSha(Transactions{})
- emptyUncleHash = CalcUncleHash(nil)
+ EmptyRootHash = DeriveSha(Transactions{})
+ EmptyUncleHash = CalcUncleHash(nil)
)
// NewBlock creates a new block. The input data is copied,
@@ -184,11 +183,11 @@ var (
// are ignored and set to values derived from the given txs, uncles
// and receipts.
func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*Receipt) *Block {
- b := &Block{header: copyHeader(header), td: new(big.Int)}
+ b := &Block{header: CopyHeader(header), td: new(big.Int)}
// TODO: panic if len(txs) != len(receipts)
if len(txs) == 0 {
- b.header.TxHash = emptyRootHash
+ b.header.TxHash = EmptyRootHash
} else {
b.header.TxHash = DeriveSha(Transactions(txs))
b.transactions = make(Transactions, len(txs))
@@ -196,21 +195,19 @@ func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*
}
if len(receipts) == 0 {
- b.header.ReceiptHash = emptyRootHash
+ b.header.ReceiptHash = EmptyRootHash
} else {
b.header.ReceiptHash = DeriveSha(Receipts(receipts))
b.header.Bloom = CreateBloom(receipts)
- b.receipts = make([]*Receipt, len(receipts))
- copy(b.receipts, receipts)
}
if len(uncles) == 0 {
- b.header.UncleHash = emptyUncleHash
+ b.header.UncleHash = EmptyUncleHash
} else {
b.header.UncleHash = CalcUncleHash(uncles)
b.uncles = make([]*Header, len(uncles))
for i := range uncles {
- b.uncles[i] = copyHeader(uncles[i])
+ b.uncles[i] = CopyHeader(uncles[i])
}
}
@@ -221,10 +218,12 @@ func NewBlock(header *Header, txs []*Transaction, uncles []*Header, receipts []*
// header data is copied, changes to header and to the field values
// will not affect the block.
func NewBlockWithHeader(header *Header) *Block {
- return &Block{header: copyHeader(header)}
+ return &Block{header: CopyHeader(header)}
}
-func copyHeader(h *Header) *Header {
+// CopyHeader creates a deep copy of a block header to prevent side effects from
+// modifying a header variable.
+func CopyHeader(h *Header) *Header {
cpy := *h
if cpy.Time = new(big.Int); h.Time != nil {
cpy.Time.Set(h.Time)
@@ -297,7 +296,6 @@ func (b *StorageBlock) DecodeRLP(s *rlp.Stream) error {
// TODO: copies
func (b *Block) Uncles() []*Header { return b.uncles }
func (b *Block) Transactions() Transactions { return b.transactions }
-func (b *Block) Receipts() Receipts { return b.receipts }
func (b *Block) Transaction(hash common.Hash) *Transaction {
for _, transaction := range b.transactions {
@@ -326,7 +324,7 @@ func (b *Block) ReceiptHash() common.Hash { return b.header.ReceiptHash }
func (b *Block) UncleHash() common.Hash { return b.header.UncleHash }
func (b *Block) Extra() []byte { return common.CopyBytes(b.header.Extra) }
-func (b *Block) Header() *Header { return copyHeader(b.header) }
+func (b *Block) Header() *Header { return CopyHeader(b.header) }
func (b *Block) HashNoNonce() common.Hash {
return b.header.HashNoNonce()
@@ -362,7 +360,6 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block {
return &Block{
header: &cpy,
transactions: b.transactions,
- receipts: b.receipts,
uncles: b.uncles,
}
}
@@ -370,13 +367,13 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block {
// WithBody returns a new block with the given transaction and uncle contents.
func (b *Block) WithBody(transactions []*Transaction, uncles []*Header) *Block {
block := &Block{
- header: copyHeader(b.header),
+ header: CopyHeader(b.header),
transactions: make([]*Transaction, len(transactions)),
uncles: make([]*Header, len(uncles)),
}
copy(block.transactions, transactions)
for i := range uncles {
- block.uncles[i] = copyHeader(uncles[i])
+ block.uncles[i] = CopyHeader(uncles[i])
}
return block
}
diff --git a/core/types/bloom9.go b/core/types/bloom9.go
index 97db20ee9..cd90fd971 100644
--- a/core/types/bloom9.go
+++ b/core/types/bloom9.go
@@ -72,7 +72,7 @@ func (b Bloom) TestBytes(test []byte) bool {
func CreateBloom(receipts Receipts) Bloom {
bin := new(big.Int)
for _, receipt := range receipts {
- bin.Or(bin, LogsBloom(receipt.logs))
+ bin.Or(bin, LogsBloom(receipt.Logs))
}
return BytesToBloom(bin.Bytes())
diff --git a/core/types/common.go b/core/types/common.go
index 29019a1b4..fe682f98a 100644
--- a/core/types/common.go
+++ b/core/types/common.go
@@ -20,4 +20,6 @@ import "github.com/ethereum/go-ethereum/core/vm"
type BlockProcessor interface {
Process(*Block) (vm.Logs, Receipts, error)
+ ValidateHeader(*Header, bool, bool) error
+ ValidateHeaderWithParent(*Header, *Header, bool, bool) error
}
diff --git a/core/types/receipt.go b/core/types/receipt.go
index bcb4bd8a5..e7d5203a3 100644
--- a/core/types/receipt.go
+++ b/core/types/receipt.go
@@ -17,7 +17,6 @@
package types
import (
- "bytes"
"fmt"
"io"
"math/big"
@@ -27,89 +26,116 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
+// Receipt represents the results of a transaction.
type Receipt struct {
+ // Consensus fields
PostState []byte
CumulativeGasUsed *big.Int
Bloom Bloom
- TxHash common.Hash
- ContractAddress common.Address
- logs vm.Logs
- GasUsed *big.Int
+ Logs vm.Logs
+
+ // Implementation fields
+ TxHash common.Hash
+ ContractAddress common.Address
+ GasUsed *big.Int
}
-func NewReceipt(root []byte, cumalativeGasUsed *big.Int) *Receipt {
- return &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: new(big.Int).Set(cumalativeGasUsed)}
+// NewReceipt creates a barebone transaction receipt, copying the init fields.
+func NewReceipt(root []byte, cumulativeGasUsed *big.Int) *Receipt {
+ return &Receipt{PostState: common.CopyBytes(root), CumulativeGasUsed: new(big.Int).Set(cumulativeGasUsed)}
}
-func (self *Receipt) SetLogs(logs vm.Logs) {
- self.logs = logs
+// EncodeRLP implements rlp.Encoder, and flattens the consensus fields of a receipt
+// into an RLP stream.
+func (r *Receipt) EncodeRLP(w io.Writer) error {
+ return rlp.Encode(w, []interface{}{r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs})
}
-func (self *Receipt) Logs() vm.Logs {
- return self.logs
+// DecodeRLP implements rlp.Decoder, and loads the consensus fields of a receipt
+// from an RLP stream.
+func (r *Receipt) DecodeRLP(s *rlp.Stream) error {
+ var receipt struct {
+ PostState []byte
+ CumulativeGasUsed *big.Int
+ Bloom Bloom
+ Logs vm.Logs
+ }
+ if err := s.Decode(&receipt); err != nil {
+ return err
+ }
+ r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs = receipt.PostState, receipt.CumulativeGasUsed, receipt.Bloom, receipt.Logs
+ return nil
}
-func (self *Receipt) EncodeRLP(w io.Writer) error {
- return rlp.Encode(w, []interface{}{self.PostState, self.CumulativeGasUsed, self.Bloom, self.logs})
+// RlpEncode implements common.RlpEncode required for SHA3 derivation.
+func (r *Receipt) RlpEncode() []byte {
+ bytes, err := rlp.EncodeToBytes(r)
+ if err != nil {
+ panic(err)
+ }
+ return bytes
}
-func (self *Receipt) DecodeRLP(s *rlp.Stream) error {
- var r struct {
+// String implements the Stringer interface.
+func (r *Receipt) String() string {
+ return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", r.PostState, r.CumulativeGasUsed, r.Bloom, r.Logs)
+}
+
+// ReceiptForStorage is a wrapper around a Receipt that flattens and parses the
+// entire content of a receipt, as opposed to only the consensus fields originally.
+type ReceiptForStorage Receipt
+
+// EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt
+// into an RLP stream.
+func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
+ logs := make([]*vm.LogForStorage, len(r.Logs))
+ for i, log := range r.Logs {
+ logs[i] = (*vm.LogForStorage)(log)
+ }
+ return rlp.Encode(w, []interface{}{r.PostState, r.CumulativeGasUsed, r.Bloom, r.TxHash, r.ContractAddress, logs, r.GasUsed})
+}
+
+// DecodeRLP implements rlp.Decoder, and loads both consensus and implementation
+// fields of a receipt from an RLP stream.
+func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
+ var receipt struct {
PostState []byte
CumulativeGasUsed *big.Int
Bloom Bloom
TxHash common.Hash
ContractAddress common.Address
- Logs vm.Logs
+ Logs []*vm.LogForStorage
GasUsed *big.Int
}
- if err := s.Decode(&r); err != nil {
+ if err := s.Decode(&receipt); err != nil {
return err
}
- self.PostState, self.CumulativeGasUsed, self.Bloom, self.TxHash, self.ContractAddress, self.logs, self.GasUsed = r.PostState, r.CumulativeGasUsed, r.Bloom, r.TxHash, r.ContractAddress, r.Logs, r.GasUsed
+ // Assign the consensus fields
+ r.PostState, r.CumulativeGasUsed, r.Bloom = receipt.PostState, receipt.CumulativeGasUsed, receipt.Bloom
+ r.Logs = make(vm.Logs, len(receipt.Logs))
+ for i, log := range receipt.Logs {
+ r.Logs[i] = (*vm.Log)(log)
+ }
+ // Assign the implementation fields
+ r.TxHash, r.ContractAddress, r.GasUsed = receipt.TxHash, receipt.ContractAddress, receipt.GasUsed
return nil
}
-type ReceiptForStorage Receipt
-
-func (self *ReceiptForStorage) EncodeRLP(w io.Writer) error {
- storageLogs := make([]*vm.LogForStorage, len(self.logs))
- for i, log := range self.logs {
- storageLogs[i] = (*vm.LogForStorage)(log)
- }
- return rlp.Encode(w, []interface{}{self.PostState, self.CumulativeGasUsed, self.Bloom, self.TxHash, self.ContractAddress, storageLogs, self.GasUsed})
-}
-
-func (self *Receipt) RlpEncode() []byte {
- bytes, err := rlp.EncodeToBytes(self)
- if err != nil {
- fmt.Println("TMP -- RECEIPT ENCODE ERROR", err)
- }
- return bytes
-}
-
-func (self *Receipt) Cmp(other *Receipt) bool {
- if bytes.Compare(self.PostState, other.PostState) != 0 {
- return false
- }
-
- return true
-}
-
-func (self *Receipt) String() string {
- return fmt.Sprintf("receipt{med=%x cgas=%v bloom=%x logs=%v}", self.PostState, self.CumulativeGasUsed, self.Bloom, self.logs)
-}
-
+// Receipts is a wrapper around a Receipt array to implement types.DerivableList.
type Receipts []*Receipt
-func (self Receipts) RlpEncode() []byte {
- bytes, err := rlp.EncodeToBytes(self)
+// RlpEncode implements common.RlpEncode required for SHA3 derivation.
+func (r Receipts) RlpEncode() []byte {
+ bytes, err := rlp.EncodeToBytes(r)
if err != nil {
- fmt.Println("TMP -- RECEIPTS ENCODE ERROR", err)
+ panic(err)
}
return bytes
}
-func (self Receipts) Len() int { return len(self) }
-func (self Receipts) GetRlp(i int) []byte { return common.Rlp(self[i]) }
+// Len returns the number of receipts in this list.
+func (r Receipts) Len() int { return len(r) }
+
+// GetRlp returns the RLP encoding of one receipt from the list.
+func (r Receipts) GetRlp(i int) []byte { return common.Rlp(r[i]) }
diff --git a/core/vm/log.go b/core/vm/log.go
index 354f0ad35..191e3a253 100644
--- a/core/vm/log.go
+++ b/core/vm/log.go
@@ -25,42 +25,47 @@ import (
)
type Log struct {
+ // Consensus fields
Address common.Address
Topics []common.Hash
Data []byte
- Number uint64
- TxHash common.Hash
- TxIndex uint
- BlockHash common.Hash
- Index uint
+ // Derived fields (don't reorder!)
+ BlockNumber uint64
+ TxHash common.Hash
+ TxIndex uint
+ BlockHash common.Hash
+ Index uint
}
func NewLog(address common.Address, topics []common.Hash, data []byte, number uint64) *Log {
- return &Log{Address: address, Topics: topics, Data: data, Number: number}
+ return &Log{Address: address, Topics: topics, Data: data, BlockNumber: number}
}
-func (self *Log) EncodeRLP(w io.Writer) error {
- return rlp.Encode(w, []interface{}{self.Address, self.Topics, self.Data})
+func (l *Log) EncodeRLP(w io.Writer) error {
+ return rlp.Encode(w, []interface{}{l.Address, l.Topics, l.Data})
}
-func (self *Log) String() string {
- return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, self.Address, self.Topics, self.Data, self.TxHash, self.TxIndex, self.BlockHash, self.Index)
+func (l *Log) DecodeRLP(s *rlp.Stream) error {
+ var log struct {
+ Address common.Address
+ Topics []common.Hash
+ Data []byte
+ }
+ if err := s.Decode(&log); err != nil {
+ return err
+ }
+ l.Address, l.Topics, l.Data = log.Address, log.Topics, log.Data
+ return nil
+}
+
+func (l *Log) String() string {
+ return fmt.Sprintf(`log: %x %x %x %x %d %x %d`, l.Address, l.Topics, l.Data, l.TxHash, l.TxIndex, l.BlockHash, l.Index)
}
type Logs []*Log
+// LogForStorage is a wrapper around a Log that flattens and parses the entire
+// content of a log, as opposed to only the consensus fields originally (by hiding
+// the rlp interface methods).
type LogForStorage Log
-
-func (self *LogForStorage) EncodeRLP(w io.Writer) error {
- return rlp.Encode(w, []interface{}{
- self.Address,
- self.Topics,
- self.Data,
- self.Number,
- self.TxHash,
- self.TxIndex,
- self.BlockHash,
- self.Index,
- })
-}
diff --git a/eth/backend.go b/eth/backend.go
index 9ec3c1440..a4f656ecd 100644
--- a/eth/backend.go
+++ b/eth/backend.go
@@ -88,6 +88,7 @@ type Config struct {
GenesisNonce int
GenesisFile string
GenesisBlock *types.Block // used by block tests
+ FastSync bool
Olympic bool
BlockChainVersion int
@@ -390,7 +391,6 @@ func New(config *Config) (*Ethereum, error) {
if err == core.ErrNoGenesis {
return nil, fmt.Errorf(`Genesis block not found. Please supply a genesis block with the "--genesis /path/to/file" argument`)
}
-
return nil, err
}
newPool := core.NewTxPool(eth.EventMux(), eth.blockchain.State, eth.blockchain.GasLimit)
@@ -398,8 +398,9 @@ func New(config *Config) (*Ethereum, error) {
eth.blockProcessor = core.NewBlockProcessor(chainDb, eth.pow, eth.blockchain, eth.EventMux())
eth.blockchain.SetProcessor(eth.blockProcessor)
- eth.protocolManager = NewProtocolManager(config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.blockchain, chainDb)
-
+ if eth.protocolManager, err = NewProtocolManager(config.FastSync, config.NetworkId, eth.eventMux, eth.txPool, eth.pow, eth.blockchain, chainDb); err != nil {
+ return nil, err
+ }
eth.miner = miner.New(eth, eth.EventMux(), eth.pow)
eth.miner.SetGasPrice(config.GasPrice)
eth.miner.SetExtra(config.ExtraData)
@@ -462,7 +463,7 @@ func (s *Ethereum) NodeInfo() *NodeInfo {
DiscPort: int(node.UDP),
TCPPort: int(node.TCP),
ListenAddr: s.net.ListenAddr,
- Td: s.BlockChain().Td().String(),
+ Td: s.BlockChain().GetTd(s.BlockChain().CurrentBlock().Hash()).String(),
}
}
diff --git a/eth/backend_test.go b/eth/backend_test.go
index 220426c17..0379fc843 100644
--- a/eth/backend_test.go
+++ b/eth/backend_test.go
@@ -16,17 +16,17 @@ func TestMipmapUpgrade(t *testing.T) {
addr := common.BytesToAddress([]byte("jeff"))
genesis := core.WriteGenesisBlockForTesting(db)
- chain := core.GenerateChain(genesis, db, 10, func(i int, gen *core.BlockGen) {
+ chain, receipts := core.GenerateChain(genesis, db, 10, func(i int, gen *core.BlockGen) {
var receipts types.Receipts
switch i {
case 1:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{&vm.Log{Address: addr}})
+ receipt.Logs = vm.Logs{&vm.Log{Address: addr}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 2:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{&vm.Log{Address: addr}})
+ receipt.Logs = vm.Logs{&vm.Log{Address: addr}}
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
}
@@ -37,7 +37,7 @@ func TestMipmapUpgrade(t *testing.T) {
t.Fatal(err)
}
})
- for _, block := range chain {
+ for i, block := range chain {
core.WriteBlock(db, block)
if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
@@ -45,7 +45,7 @@ func TestMipmapUpgrade(t *testing.T) {
if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
- if err := core.PutBlockReceipts(db, block, block.Receipts()); err != nil {
+ if err := core.PutBlockReceipts(db, block.Hash(), receipts[i]); err != nil {
t.Fatal("error writing block receipts:", err)
}
}
diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go
index 64fb1b57b..4bcbd8557 100644
--- a/eth/downloader/downloader.go
+++ b/eth/downloader/downloader.go
@@ -18,111 +18,85 @@
package downloader
import (
+ "crypto/rand"
"errors"
+ "fmt"
"math"
"math/big"
+ "strings"
"sync"
"sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
-)
-
-const (
- eth61 = 61 // Constant to check for old protocol support
- eth62 = 62 // Constant to check for new protocol support
+ "github.com/rcrowley/go-metrics"
)
var (
- MaxHashFetch = 512 // Amount of hashes to be fetched per retrieval request
- MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request
- MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request
- MaxBodyFetch = 128 // Amount of block bodies to be fetched per retrieval request
- MaxStateFetch = 384 // Amount of node state values to allow fetching per request
- MaxReceiptsFetch = 384 // Amount of transaction receipts to allow fetching per request
+ MaxHashFetch = 512 // Amount of hashes to be fetched per retrieval request
+ MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request
+ MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request
+ MaxBodyFetch = 128 // Amount of block bodies to be fetched per retrieval request
+ MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request
+ MaxStateFetch = 384 // Amount of node state values to allow fetching per request
- hashTTL = 5 * time.Second // [eth/61] Time it takes for a hash request to time out
- blockSoftTTL = 3 * time.Second // [eth/61] Request completion threshold for increasing or decreasing a peer's bandwidth
- blockHardTTL = 3 * blockSoftTTL // [eth/61] Maximum time allowance before a block request is considered expired
- headerTTL = 5 * time.Second // [eth/62] Time it takes for a header request to time out
- bodySoftTTL = 3 * time.Second // [eth/62] Request completion threshold for increasing or decreasing a peer's bandwidth
- bodyHardTTL = 3 * bodySoftTTL // [eth/62] Maximum time allowance before a block body request is considered expired
+ hashTTL = 5 * time.Second // [eth/61] Time it takes for a hash request to time out
+ blockSoftTTL = 3 * time.Second // [eth/61] Request completion threshold for increasing or decreasing a peer's bandwidth
+ blockHardTTL = 3 * blockSoftTTL // [eth/61] Maximum time allowance before a block request is considered expired
+ headerTTL = 5 * time.Second // [eth/62] Time it takes for a header request to time out
+ bodySoftTTL = 3 * time.Second // [eth/62] Request completion threshold for increasing or decreasing a peer's bandwidth
+ bodyHardTTL = 3 * bodySoftTTL // [eth/62] Maximum time allowance before a block body request is considered expired
+ receiptSoftTTL = 3 * time.Second // [eth/63] Request completion threshold for increasing or decreasing a peer's bandwidth
+ receiptHardTTL = 3 * receiptSoftTTL // [eth/63] Maximum time allowance before a receipt request is considered expired
+ stateSoftTTL = 2 * time.Second // [eth/63] Request completion threshold for increasing or decreasing a peer's bandwidth
+ stateHardTTL = 3 * stateSoftTTL // [eth/63] Maximum time allowance before a node data request is considered expired
- maxQueuedHashes = 256 * 1024 // [eth/61] Maximum number of hashes to queue for import (DOS protection)
- maxQueuedHeaders = 256 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection)
- maxBlockProcess = 256 // Number of blocks to import at once into the chain
+ maxQueuedHashes = 256 * 1024 // [eth/61] Maximum number of hashes to queue for import (DOS protection)
+ maxQueuedHeaders = 256 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection)
+ maxQueuedStates = 256 * 1024 // [eth/63] Maximum number of state requests to queue (DOS protection)
+ maxResultsProcess = 256 // Number of download results to import at once into the chain
+
+ fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync
+ fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected
+ fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it
+ fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point
+ fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync
)
var (
- errBusy = errors.New("busy")
- errUnknownPeer = errors.New("peer is unknown or unhealthy")
- errBadPeer = errors.New("action from bad peer ignored")
- errStallingPeer = errors.New("peer is stalling")
- errNoPeers = errors.New("no peers to keep download active")
- errPendingQueue = errors.New("pending items in queue")
- errTimeout = errors.New("timeout")
- errEmptyHashSet = errors.New("empty hash set by peer")
- errEmptyHeaderSet = errors.New("empty header set by peer")
- errPeersUnavailable = errors.New("no peers available or all peers tried for block download process")
- errAlreadyInPool = errors.New("hash already in pool")
- errInvalidChain = errors.New("retrieved hash chain is invalid")
- errInvalidBody = errors.New("retrieved block body is invalid")
- errCancelHashFetch = errors.New("hash fetching canceled (requested)")
- errCancelBlockFetch = errors.New("block downloading canceled (requested)")
- errCancelHeaderFetch = errors.New("block header fetching canceled (requested)")
- errCancelBodyFetch = errors.New("block body downloading canceled (requested)")
- errNoSyncActive = errors.New("no sync active")
+ errBusy = errors.New("busy")
+ errUnknownPeer = errors.New("peer is unknown or unhealthy")
+ errBadPeer = errors.New("action from bad peer ignored")
+ errStallingPeer = errors.New("peer is stalling")
+ errNoPeers = errors.New("no peers to keep download active")
+ errPendingQueue = errors.New("pending items in queue")
+ errTimeout = errors.New("timeout")
+ errEmptyHashSet = errors.New("empty hash set by peer")
+ errEmptyHeaderSet = errors.New("empty header set by peer")
+ errPeersUnavailable = errors.New("no peers available or all tried for download")
+ errAlreadyInPool = errors.New("hash already in pool")
+ errInvalidChain = errors.New("retrieved hash chain is invalid")
+ errInvalidBlock = errors.New("retrieved block is invalid")
+ errInvalidBody = errors.New("retrieved block body is invalid")
+ errInvalidReceipt = errors.New("retrieved receipt is invalid")
+ errCancelHashFetch = errors.New("hash download canceled (requested)")
+ errCancelBlockFetch = errors.New("block download canceled (requested)")
+ errCancelHeaderFetch = errors.New("block header download canceled (requested)")
+ errCancelBodyFetch = errors.New("block body download canceled (requested)")
+ errCancelReceiptFetch = errors.New("receipt download canceled (requested)")
+ errCancelStateFetch = errors.New("state data download canceled (requested)")
+ errNoSyncActive = errors.New("no sync active")
)
-// hashCheckFn is a callback type for verifying a hash's presence in the local chain.
-type hashCheckFn func(common.Hash) bool
-
-// blockRetrievalFn is a callback type for retrieving a block from the local chain.
-type blockRetrievalFn func(common.Hash) *types.Block
-
-// headRetrievalFn is a callback type for retrieving the head block from the local chain.
-type headRetrievalFn func() *types.Block
-
-// tdRetrievalFn is a callback type for retrieving the total difficulty of a local block.
-type tdRetrievalFn func(common.Hash) *big.Int
-
-// chainInsertFn is a callback type to insert a batch of blocks into the local chain.
-type chainInsertFn func(types.Blocks) (int, error)
-
-// peerDropFn is a callback type for dropping a peer detected as malicious.
-type peerDropFn func(id string)
-
-// hashPack is a batch of block hashes returned by a peer (eth/61).
-type hashPack struct {
- peerId string
- hashes []common.Hash
-}
-
-// blockPack is a batch of blocks returned by a peer (eth/61).
-type blockPack struct {
- peerId string
- blocks []*types.Block
-}
-
-// headerPack is a batch of block headers returned by a peer.
-type headerPack struct {
- peerId string
- headers []*types.Header
-}
-
-// bodyPack is a batch of block bodies returned by a peer.
-type bodyPack struct {
- peerId string
- transactions [][]*types.Transaction
- uncles [][]*types.Header
-}
-
type Downloader struct {
- mux *event.TypeMux
+ mode SyncMode // Synchronisation mode defining the strategy used (per sync cycle)
+ noFast bool // Flag to disable fast syncing in case of a security error
+ mux *event.TypeMux // Event multiplexer to announce sync operation events
queue *queue // Scheduler for selecting the hashes to download
peers *peerSet // Set of active peers from which download can proceed
@@ -130,17 +104,27 @@ type Downloader struct {
interrupt int32 // Atomic boolean to signal termination
// Statistics
- syncStatsOrigin uint64 // Origin block number where syncing started at
- syncStatsHeight uint64 // Highest block number known when syncing started
- syncStatsLock sync.RWMutex // Lock protecting the sync stats fields
+ syncStatsChainOrigin uint64 // Origin block number where syncing started at
+ syncStatsChainHeight uint64 // Highest block number known when syncing started
+ syncStatsStateTotal uint64 // Total number of node state entries known so far
+ syncStatsStateDone uint64 // Number of state trie entries already pulled
+ syncStatsLock sync.RWMutex // Lock protecting the sync stats fields
// Callbacks
- hasBlock hashCheckFn // Checks if a block is present in the chain
- getBlock blockRetrievalFn // Retrieves a block from the chain
- headBlock headRetrievalFn // Retrieves the head block from the chain
- getTd tdRetrievalFn // Retrieves the TD of a block from the chain
- insertChain chainInsertFn // Injects a batch of blocks into the chain
- dropPeer peerDropFn // Drops a peer for misbehaving
+ hasHeader headerCheckFn // Checks if a header is present in the chain
+ hasBlock blockCheckFn // Checks if a block is present in the chain
+ getHeader headerRetrievalFn // Retrieves a header from the chain
+ getBlock blockRetrievalFn // Retrieves a block from the chain
+ headHeader headHeaderRetrievalFn // Retrieves the head header from the chain
+ headBlock headBlockRetrievalFn // Retrieves the head block from the chain
+ headFastBlock headFastBlockRetrievalFn // Retrieves the head fast-sync block from the chain
+ commitHeadBlock headBlockCommitterFn // Commits a manually assembled block as the chain head
+ getTd tdRetrievalFn // Retrieves the TD of a block from the chain
+ insertHeaders headerChainInsertFn // Injects a batch of headers into the chain
+ insertBlocks blockChainInsertFn // Injects a batch of blocks into the chain
+ insertReceipts receiptChainInsertFn // Injects a batch of blocks and their receipts into the chain
+ rollback chainRollbackFn // Removes a batch of recently added chain links
+ dropPeer peerDropFn // Drops a peer for misbehaving
// Status
synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing
@@ -149,72 +133,100 @@ type Downloader struct {
notified int32
// Channels
- newPeerCh chan *peer
- hashCh chan hashPack // [eth/61] Channel receiving inbound hashes
- blockCh chan blockPack // [eth/61] Channel receiving inbound blocks
- headerCh chan headerPack // [eth/62] Channel receiving inbound block headers
- bodyCh chan bodyPack // [eth/62] Channel receiving inbound block bodies
- wakeCh chan bool // Channel to signal the block/body fetcher of new tasks
+ newPeerCh chan *peer
+ hashCh chan dataPack // [eth/61] Channel receiving inbound hashes
+ blockCh chan dataPack // [eth/61] Channel receiving inbound blocks
+ headerCh chan dataPack // [eth/62] Channel receiving inbound block headers
+ bodyCh chan dataPack // [eth/62] Channel receiving inbound block bodies
+ receiptCh chan dataPack // [eth/63] Channel receiving inbound receipts
+ stateCh chan dataPack // [eth/63] Channel receiving inbound node state data
+ blockWakeCh chan bool // [eth/61] Channel to signal the block fetcher of new tasks
+ bodyWakeCh chan bool // [eth/62] Channel to signal the block body fetcher of new tasks
+ receiptWakeCh chan bool // [eth/63] Channel to signal the receipt fetcher of new tasks
+ stateWakeCh chan bool // [eth/63] Channel to signal the state fetcher of new tasks
cancelCh chan struct{} // Channel to cancel mid-flight syncs
cancelLock sync.RWMutex // Lock to protect the cancel channel in delivers
// Testing hooks
- syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run
- bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch
- chainInsertHook func([]*Block) // Method to call upon inserting a chain of blocks (possibly in multiple invocations)
-}
-
-// Block is an origin-tagged blockchain block.
-type Block struct {
- RawBlock *types.Block
- OriginPeer string
+ syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run
+ bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch
+ receiptFetchHook func([]*types.Header) // Method to call upon starting a receipt fetch
+ chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations)
}
// New creates a new downloader to fetch hashes and blocks from remote peers.
-func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, headBlock headRetrievalFn, getTd tdRetrievalFn, insertChain chainInsertFn, dropPeer peerDropFn) *Downloader {
+func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlock blockCheckFn, getHeader headerRetrievalFn,
+ getBlock blockRetrievalFn, headHeader headHeaderRetrievalFn, headBlock headBlockRetrievalFn, headFastBlock headFastBlockRetrievalFn,
+ commitHeadBlock headBlockCommitterFn, getTd tdRetrievalFn, insertHeaders headerChainInsertFn, insertBlocks blockChainInsertFn,
+ insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn) *Downloader {
+
return &Downloader{
- mux: mux,
- queue: newQueue(),
- peers: newPeerSet(),
- hasBlock: hasBlock,
- getBlock: getBlock,
- headBlock: headBlock,
- getTd: getTd,
- insertChain: insertChain,
- dropPeer: dropPeer,
- newPeerCh: make(chan *peer, 1),
- hashCh: make(chan hashPack, 1),
- blockCh: make(chan blockPack, 1),
- headerCh: make(chan headerPack, 1),
- bodyCh: make(chan bodyPack, 1),
- wakeCh: make(chan bool, 1),
+ mode: FullSync,
+ mux: mux,
+ queue: newQueue(stateDb),
+ peers: newPeerSet(),
+ hasHeader: hasHeader,
+ hasBlock: hasBlock,
+ getHeader: getHeader,
+ getBlock: getBlock,
+ headHeader: headHeader,
+ headBlock: headBlock,
+ headFastBlock: headFastBlock,
+ commitHeadBlock: commitHeadBlock,
+ getTd: getTd,
+ insertHeaders: insertHeaders,
+ insertBlocks: insertBlocks,
+ insertReceipts: insertReceipts,
+ rollback: rollback,
+ dropPeer: dropPeer,
+ newPeerCh: make(chan *peer, 1),
+ hashCh: make(chan dataPack, 1),
+ blockCh: make(chan dataPack, 1),
+ headerCh: make(chan dataPack, 1),
+ bodyCh: make(chan dataPack, 1),
+ receiptCh: make(chan dataPack, 1),
+ stateCh: make(chan dataPack, 1),
+ blockWakeCh: make(chan bool, 1),
+ bodyWakeCh: make(chan bool, 1),
+ receiptWakeCh: make(chan bool, 1),
+ stateWakeCh: make(chan bool, 1),
}
}
-// Boundaries retrieves the synchronisation boundaries, specifically the origin
-// block where synchronisation started at (may have failed/suspended) and the
-// latest known block which the synchonisation targets.
-func (d *Downloader) Boundaries() (uint64, uint64) {
+// Progress retrieves the synchronisation boundaries, specifically the origin
+// block where synchronisation started at (may have failed/suspended); the block
+// or header sync is currently at; and the latest known block which the sync targets.
+func (d *Downloader) Progress() (uint64, uint64, uint64) {
d.syncStatsLock.RLock()
defer d.syncStatsLock.RUnlock()
- return d.syncStatsOrigin, d.syncStatsHeight
+ current := uint64(0)
+ switch d.mode {
+ case FullSync:
+ current = d.headBlock().NumberU64()
+ case FastSync:
+ current = d.headFastBlock().NumberU64()
+ case LightSync:
+ current = d.headHeader().Number.Uint64()
+ }
+ return d.syncStatsChainOrigin, current, d.syncStatsChainHeight
}
// Synchronising returns whether the downloader is currently retrieving blocks.
func (d *Downloader) Synchronising() bool {
- return atomic.LoadInt32(&d.synchronising) > 0
+ return atomic.LoadInt32(&d.synchronising) > 0 || atomic.LoadInt32(&d.processing) > 0
}
// RegisterPeer injects a new download peer into the set of block source to be
// used for fetching hashes and blocks from.
func (d *Downloader) RegisterPeer(id string, version int, head common.Hash,
getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading
- getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn) error {
+ getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn,
+ getReceipts receiptFetcherFn, getNodeData stateFetcherFn) error {
glog.V(logger.Detail).Infoln("Registering peer", id)
- if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks, getRelHeaders, getAbsHeaders, getBlockBodies)); err != nil {
+ if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil {
glog.V(logger.Error).Infoln("Register failed:", err)
return err
}
@@ -222,22 +234,24 @@ func (d *Downloader) RegisterPeer(id string, version int, head common.Hash,
}
// UnregisterPeer remove a peer from the known list, preventing any action from
-// the specified peer.
+// the specified peer. An effort is also made to return any pending fetches into
+// the queue.
func (d *Downloader) UnregisterPeer(id string) error {
glog.V(logger.Detail).Infoln("Unregistering peer", id)
if err := d.peers.Unregister(id); err != nil {
glog.V(logger.Error).Infoln("Unregister failed:", err)
return err
}
+ d.queue.Revoke(id)
return nil
}
// Synchronise tries to sync up our local block chain with a remote peer, both
// adding various sanity checks as well as wrapping it with various log entries.
-func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
+func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode SyncMode) {
glog.V(logger.Detail).Infof("Attempting synchronisation: %v, head [%x…], TD %v", id, head[:4], td)
- switch err := d.synchronise(id, head, td); err {
+ switch err := d.synchronise(id, head, td, mode); err {
case nil:
glog.V(logger.Detail).Infof("Synchronisation completed")
@@ -259,7 +273,7 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
// synchronise will select the peer and use it for synchronising. If an empty string is given
// it will use the best peer possible and synchronize if it's TD is higher than our own. If any of the
// checks fail an error will be returned. This method is synchronous
-func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error {
+func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode SyncMode) error {
// Mock out the synchonisation if testing
if d.synchroniseMock != nil {
return d.synchroniseMock(id, hash)
@@ -275,22 +289,35 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
glog.V(logger.Info).Infoln("Block synchronisation started")
}
// Abort if the queue still contains some leftover data
- if _, cached := d.queue.Size(); cached > 0 && d.queue.GetHeadBlock() != nil {
+ if d.queue.GetHeadResult() != nil {
return errPendingQueue
}
- // Reset the queue and peer set to clean any internal leftover state
+ // Reset the queue, peer set and wake channels to clean any internal leftover state
d.queue.Reset()
d.peers.Reset()
- select {
- case <-d.wakeCh:
- default:
+ for _, ch := range []chan bool{d.blockWakeCh, d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} {
+ select {
+ case <-ch:
+ default:
+ }
}
+ // Reset and ephemeral sync statistics
+ d.syncStatsLock.Lock()
+ d.syncStatsStateTotal = 0
+ d.syncStatsStateDone = 0
+ d.syncStatsLock.Unlock()
+
// Create cancel channel for aborting mid-flight
d.cancelLock.Lock()
d.cancelCh = make(chan struct{})
d.cancelLock.Unlock()
+ // Set the requested sync mode, unless it's forbidden
+ d.mode = mode
+ if d.mode == FastSync && d.noFast {
+ d.mode = FullSync
+ }
// Retrieve the origin peer and initiate the downloading process
p := d.peers.Peer(id)
if p == nil {
@@ -299,12 +326,6 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
return d.syncWithPeer(p, hash, td)
}
-// Has checks if the downloader knows about a particular hash, meaning that its
-// either already downloaded of pending retrieval.
-func (d *Downloader) Has(hash common.Hash) bool {
- return d.queue.Has(hash)
-}
-
// syncWithPeer starts a block synchronization based on the hash chain from the
// specified peer and head hash.
func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err error) {
@@ -320,10 +341,12 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
}()
glog.V(logger.Debug).Infof("Synchronising with the network using: %s [eth/%d]", p.id, p.version)
- defer glog.V(logger.Debug).Infof("Synchronisation terminated")
+ defer func(start time.Time) {
+ glog.V(logger.Debug).Infof("Synchronisation terminated after %v", time.Since(start))
+ }(time.Now())
switch {
- case p.version == eth61:
+ case p.version == 61:
// Look up the sync boundaries: the common ancestor and the target block
latest, err := d.fetchHeight61(p)
if err != nil {
@@ -334,16 +357,18 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
return err
}
d.syncStatsLock.Lock()
- if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin {
- d.syncStatsOrigin = origin
+ if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin {
+ d.syncStatsChainOrigin = origin
}
- d.syncStatsHeight = latest
+ d.syncStatsChainHeight = latest
d.syncStatsLock.Unlock()
// Initiate the sync using a concurrent hash and block retrieval algorithm
if d.syncInitHook != nil {
d.syncInitHook(origin, latest)
}
+ d.queue.Prepare(origin+1, d.mode, 0)
+
errc := make(chan error, 2)
go func() { errc <- d.fetchHashes61(p, td, origin+1) }()
go func() { errc <- d.fetchBlocks61(origin + 1) }()
@@ -356,7 +381,7 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
}
return <-errc
- case p.version >= eth62:
+ case p.version >= 62:
// Look up the sync boundaries: the common ancestor and the target block
latest, err := d.fetchHeight(p)
if err != nil {
@@ -367,27 +392,59 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
return err
}
d.syncStatsLock.Lock()
- if d.syncStatsHeight <= origin || d.syncStatsOrigin > origin {
- d.syncStatsOrigin = origin
+ if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin {
+ d.syncStatsChainOrigin = origin
}
- d.syncStatsHeight = latest
+ d.syncStatsChainHeight = latest
d.syncStatsLock.Unlock()
- // Initiate the sync using a concurrent hash and block retrieval algorithm
+ // Initiate the sync using a concurrent header and content retrieval algorithm
+ pivot := uint64(0)
+ switch d.mode {
+ case LightSync:
+ pivot = latest
+
+ case FastSync:
+ // Calculate the new fast/slow sync pivot point
+ pivotOffset, err := rand.Int(rand.Reader, big.NewInt(int64(fsPivotInterval)))
+ if err != nil {
+ panic(fmt.Sprintf("Failed to access crypto random source: %v", err))
+ }
+ if latest > uint64(fsMinFullBlocks)+pivotOffset.Uint64() {
+ pivot = latest - uint64(fsMinFullBlocks) - pivotOffset.Uint64()
+ }
+ // If the point is below the origin, move origin back to ensure state download
+ if pivot < origin {
+ if pivot > 0 {
+ origin = pivot - 1
+ } else {
+ origin = 0
+ }
+ }
+ glog.V(logger.Debug).Infof("Fast syncing until pivot block #%d", pivot)
+ }
+ d.queue.Prepare(origin+1, d.mode, pivot)
+
if d.syncInitHook != nil {
d.syncInitHook(origin, latest)
}
- errc := make(chan error, 2)
- go func() { errc <- d.fetchHeaders(p, td, origin+1) }()
- go func() { errc <- d.fetchBodies(origin + 1) }()
+ errc := make(chan error, 4)
+ go func() { errc <- d.fetchHeaders(p, td, origin+1) }() // Headers are always retrieved
+ go func() { errc <- d.fetchBodies(origin + 1) }() // Bodies are retrieved during normal and fast sync
+ go func() { errc <- d.fetchReceipts(origin + 1) }() // Receipts are retrieved during fast sync
+ go func() { errc <- d.fetchNodeData() }() // Node state data is retrieved during fast sync
- // If any fetcher fails, cancel the other
- if err := <-errc; err != nil {
- d.cancel()
- <-errc
- return err
+ // If any fetcher fails, cancel the others
+ var fail error
+ for i := 0; i < cap(errc); i++ {
+ if err := <-errc; err != nil {
+ if fail == nil {
+ fail = err
+ d.cancel()
+ }
+ }
}
- return <-errc
+ return fail
default:
// Something very wrong, stop right here
@@ -445,14 +502,14 @@ func (d *Downloader) fetchHeight61(p *peer) (uint64, error) {
case <-d.hashCh:
// Out of bounds hashes received, ignore them
- case blockPack := <-d.blockCh:
+ case packet := <-d.blockCh:
// Discard anything not from the origin peer
- if blockPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", blockPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received blocks from incorrect peer(%s)", packet.PeerId())
break
}
// Make sure the peer actually gave something valid
- blocks := blockPack.blocks
+ blocks := packet.(*blockPack).blocks
if len(blocks) != 1 {
glog.V(logger.Debug).Infof("%v: invalid number of head blocks: %d != 1", p, len(blocks))
return 0, errBadPeer
@@ -491,14 +548,14 @@ func (d *Downloader) findAncestor61(p *peer) (uint64, error) {
case <-d.cancelCh:
return 0, errCancelHashFetch
- case hashPack := <-d.hashCh:
+ case packet := <-d.hashCh:
// Discard anything not from the origin peer
- if hashPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", hashPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId())
break
}
// Make sure the peer actually gave something valid
- hashes := hashPack.hashes
+ hashes := packet.(*hashPack).hashes
if len(hashes) == 0 {
glog.V(logger.Debug).Infof("%v: empty head hash set", p)
return 0, errEmptyHashSet
@@ -546,14 +603,14 @@ func (d *Downloader) findAncestor61(p *peer) (uint64, error) {
case <-d.cancelCh:
return 0, errCancelHashFetch
- case hashPack := <-d.hashCh:
+ case packet := <-d.hashCh:
// Discard anything not from the origin peer
- if hashPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", hashPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId())
break
}
// Make sure the peer actually gave something valid
- hashes := hashPack.hashes
+ hashes := packet.(*hashPack).hashes
if len(hashes) != 1 {
glog.V(logger.Debug).Infof("%v: invalid search hash set (%d)", p, len(hashes))
return 0, errBadPeer
@@ -623,21 +680,21 @@ func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error {
case <-d.bodyCh:
// Out of bounds eth/62 block bodies received, ignore them
- case hashPack := <-d.hashCh:
+ case packet := <-d.hashCh:
// Make sure the active peer is giving us the hashes
- if hashPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", hashPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received hashes from incorrect peer(%s)", packet.PeerId())
break
}
hashReqTimer.UpdateSince(request)
timeout.Stop()
// If no more hashes are inbound, notify the block fetcher and return
- if len(hashPack.hashes) == 0 {
+ if packet.Items() == 0 {
glog.V(logger.Debug).Infof("%v: no available hashes", p)
select {
- case d.wakeCh <- false:
+ case d.blockWakeCh <- false:
case <-d.cancelCh:
}
// If no hashes were retrieved at all, the peer violated it's TD promise that it had a
@@ -658,32 +715,33 @@ func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error {
return nil
}
gotHashes = true
+ hashes := packet.(*hashPack).hashes
// Otherwise insert all the new hashes, aborting in case of junk
- glog.V(logger.Detail).Infof("%v: inserting %d hashes from #%d", p, len(hashPack.hashes), from)
+ glog.V(logger.Detail).Infof("%v: scheduling %d hashes from #%d", p, len(hashes), from)
- inserts := d.queue.Insert61(hashPack.hashes, true)
- if len(inserts) != len(hashPack.hashes) {
+ inserts := d.queue.Schedule61(hashes, true)
+ if len(inserts) != len(hashes) {
glog.V(logger.Debug).Infof("%v: stale hashes", p)
return errBadPeer
}
// Notify the block fetcher of new hashes, but stop if queue is full
- if d.queue.Pending() < maxQueuedHashes {
+ if d.queue.PendingBlocks() < maxQueuedHashes {
// We still have hashes to fetch, send continuation wake signal (potential)
select {
- case d.wakeCh <- true:
+ case d.blockWakeCh <- true:
default:
}
} else {
// Hash limit reached, send a termination wake signal (enforced)
select {
- case d.wakeCh <- false:
+ case d.blockWakeCh <- false:
case <-d.cancelCh:
}
return nil
}
// Queue not yet full, fetch the next batch
- from += uint64(len(hashPack.hashes))
+ from += uint64(len(hashes))
getHashes(from)
case <-timeout.C:
@@ -707,10 +765,8 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
update := make(chan struct{}, 1)
- // Prepare the queue and fetch blocks until the hash fetcher's done
- d.queue.Prepare(from)
+ // Fetch blocks until the hash fetcher's done
finished := false
-
for {
select {
case <-d.cancelCh:
@@ -722,25 +778,26 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
case <-d.bodyCh:
// Out of bounds eth/62 block bodies received, ignore them
- case blockPack := <-d.blockCh:
+ case packet := <-d.blockCh:
// If the peer was previously banned and failed to deliver it's pack
// in a reasonable time frame, ignore it's message.
- if peer := d.peers.Peer(blockPack.peerId); peer != nil {
+ if peer := d.peers.Peer(packet.PeerId()); peer != nil {
// Deliver the received chunk of blocks, and demote in case of errors
- err := d.queue.Deliver61(blockPack.peerId, blockPack.blocks)
+ blocks := packet.(*blockPack).blocks
+ err := d.queue.DeliverBlocks(peer.id, blocks)
switch err {
case nil:
// If no blocks were delivered, demote the peer (need the delivery above)
- if len(blockPack.blocks) == 0 {
+ if len(blocks) == 0 {
peer.Demote()
- peer.SetIdle61()
+ peer.SetBlocksIdle()
glog.V(logger.Detail).Infof("%s: no blocks delivered", peer)
break
}
// All was successful, promote the peer and potentially start processing
peer.Promote()
- peer.SetIdle61()
- glog.V(logger.Detail).Infof("%s: delivered %d blocks", peer, len(blockPack.blocks))
+ peer.SetBlocksIdle()
+ glog.V(logger.Detail).Infof("%s: delivered %d blocks", peer, len(blocks))
go d.process()
case errInvalidChain:
@@ -751,7 +808,7 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
// Peer probably timed out with its delivery but came through
// in the end, demote, but allow to to pull from this peer.
peer.Demote()
- peer.SetIdle61()
+ peer.SetBlocksIdle()
glog.V(logger.Detail).Infof("%s: out of bound delivery", peer)
case errStaleDelivery:
@@ -765,7 +822,7 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
default:
// Peer did something semi-useful, demote but keep it around
peer.Demote()
- peer.SetIdle61()
+ peer.SetBlocksIdle()
glog.V(logger.Detail).Infof("%s: delivery partially failed: %v", peer, err)
go d.process()
}
@@ -776,7 +833,7 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
default:
}
- case cont := <-d.wakeCh:
+ case cont := <-d.blockWakeCh:
// The hash fetcher sent a continuation flag, check if it's done
if !cont {
finished = true
@@ -800,15 +857,15 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
return errNoPeers
}
// Check for block request timeouts and demote the responsible peers
- for _, pid := range d.queue.Expire(blockHardTTL) {
+ for _, pid := range d.queue.ExpireBlocks(blockHardTTL) {
if peer := d.peers.Peer(pid); peer != nil {
peer.Demote()
glog.V(logger.Detail).Infof("%s: block delivery timeout", peer)
}
}
- // If there's noting more to fetch, wait or terminate
- if d.queue.Pending() == 0 {
- if d.queue.InFlight() == 0 && finished {
+ // If there's nothing more to fetch, wait or terminate
+ if d.queue.PendingBlocks() == 0 {
+ if !d.queue.InFlightBlocks() && finished {
glog.V(logger.Debug).Infof("Block fetching completed")
return nil
}
@@ -816,16 +873,18 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
}
// Send a download request to all idle peers, until throttled
throttled := false
- for _, peer := range d.peers.IdlePeers(eth61) {
+ idles, total := d.peers.BlockIdlePeers()
+
+ for _, peer := range idles {
// Short circuit if throttling activated
- if d.queue.Throttle() {
+ if d.queue.ShouldThrottleBlocks() {
throttled = true
break
}
// Reserve a chunk of hashes for a peer. A nil can mean either that
// no more hashes are available, or that the peer is known not to
// have them.
- request := d.queue.Reserve61(peer, peer.Capacity())
+ request := d.queue.ReserveBlocks(peer, peer.BlockCapacity())
if request == nil {
continue
}
@@ -834,13 +893,18 @@ func (d *Downloader) fetchBlocks61(from uint64) error {
}
// Fetch the chunk and make sure any errors return the hashes to the queue
if err := peer.Fetch61(request); err != nil {
- glog.V(logger.Error).Infof("%v: fetch failed, rescheduling", peer)
- d.queue.Cancel(request)
+ // Although we could try and make an attempt to fix this, this error really
+ // means that we've double allocated a fetch task to a peer. If that is the
+ // case, the internal state of the downloader and the queue is very wrong so
+ // better hard crash and note the error instead of silently accumulating into
+ // a much bigger issue.
+ panic(fmt.Sprintf("%v: fetch assignment failed, hard panic", peer))
+ d.queue.CancelBlocks(request) // noop for now
}
}
// Make sure that we have peers available for fetching. If all peers have been tried
// and all failed throw an error
- if !throttled && d.queue.InFlight() == 0 {
+ if !throttled && !d.queue.InFlightBlocks() && len(idles) == total {
return errPeersUnavailable
}
}
@@ -861,14 +925,14 @@ func (d *Downloader) fetchHeight(p *peer) (uint64, error) {
case <-d.cancelCh:
return 0, errCancelBlockFetch
- case headerPack := <-d.headerCh:
+ case packet := <-d.headerCh:
// Discard anything not from the origin peer
- if headerPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", packet.PeerId())
break
}
// Make sure the peer actually gave something valid
- headers := headerPack.headers
+ headers := packet.(*headerPack).headers
if len(headers) != 1 {
glog.V(logger.Debug).Infof("%v: invalid number of head headers: %d != 1", p, len(headers))
return 0, errBadPeer
@@ -891,16 +955,21 @@ func (d *Downloader) fetchHeight(p *peer) (uint64, error) {
}
}
-// findAncestor tries to locate the common ancestor block of the local chain and
+// findAncestor tries to locate the common ancestor link of the local chain and
// a remote peers blockchain. In the general case when our node was in sync and
-// on the correct chain, checking the top N blocks should already get us a match.
+// on the correct chain, checking the top N links should already get us a match.
// In the rare scenario when we ended up on a long reorganization (i.e. none of
-// the head blocks match), we do a binary search to find the common ancestor.
+// the head links match), we do a binary search to find the common ancestor.
func (d *Downloader) findAncestor(p *peer) (uint64, error) {
glog.V(logger.Debug).Infof("%v: looking for common ancestor", p)
- // Request our head blocks to short circuit ancestor location
- head := d.headBlock().NumberU64()
+ // Request our head headers to short circuit ancestor location
+ head := d.headHeader().Number.Uint64()
+ if d.mode == FullSync {
+ head = d.headBlock().NumberU64()
+ } else if d.mode == FastSync {
+ head = d.headFastBlock().NumberU64()
+ }
from := int64(head) - int64(MaxHeaderFetch) + 1
if from < 0 {
from = 0
@@ -916,14 +985,14 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
case <-d.cancelCh:
return 0, errCancelHashFetch
- case headerPack := <-d.headerCh:
+ case packet := <-d.headerCh:
// Discard anything not from the origin peer
- if headerPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", packet.PeerId())
break
}
// Make sure the peer actually gave something valid
- headers := headerPack.headers
+ headers := packet.(*headerPack).headers
if len(headers) == 0 {
glog.V(logger.Debug).Infof("%v: empty head header set", p)
return 0, errEmptyHeaderSet
@@ -931,7 +1000,7 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
// Check if a common ancestor was found
finished = true
for i := len(headers) - 1; i >= 0; i-- {
- if d.hasBlock(headers[i].Hash()) {
+ if (d.mode != LightSync && d.hasBlock(headers[i].Hash())) || (d.mode == LightSync && d.hasHeader(headers[i].Hash())) {
number, hash = headers[i].Number.Uint64(), headers[i].Hash()
break
}
@@ -971,14 +1040,14 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
case <-d.cancelCh:
return 0, errCancelHashFetch
- case headerPack := <-d.headerCh:
+ case packer := <-d.headerCh:
// Discard anything not from the origin peer
- if headerPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
+ if packer.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", packer.PeerId())
break
}
// Make sure the peer actually gave something valid
- headers := headerPack.headers
+ headers := packer.(*headerPack).headers
if len(headers) != 1 {
glog.V(logger.Debug).Infof("%v: invalid search header set (%d)", p, len(headers))
return 0, errBadPeer
@@ -986,13 +1055,13 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
arrived = true
// Modify the search interval based on the response
- block := d.getBlock(headers[0].Hash())
- if block == nil {
+ if (d.mode == FullSync && !d.hasBlock(headers[0].Hash())) || (d.mode != FullSync && !d.hasHeader(headers[0].Hash())) {
end = check
break
}
- if block.NumberU64() != check {
- glog.V(logger.Debug).Infof("%v: non requested header #%d [%x…], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check)
+ header := d.getHeader(headers[0].Hash()) // Independent of sync mode, header surely exists
+ if header.Number.Uint64() != check {
+ glog.V(logger.Debug).Infof("%v: non requested header #%d [%x…], instead of #%d", p, header.Number, header.Hash().Bytes()[:4], check)
return 0, errBadPeer
}
start = check
@@ -1017,10 +1086,37 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
// fetchHeaders keeps retrieving headers from the requested number, until no more
// are returned, potentially throttling on the way.
+//
+// The queue parameter can be used to switch between queuing headers for block
+// body download too, or directly import as pure header chains.
func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
glog.V(logger.Debug).Infof("%v: downloading headers from #%d", p, from)
defer glog.V(logger.Debug).Infof("%v: header download terminated", p)
+ // Calculate the pivoting point for switching from fast to slow sync
+ pivot := d.queue.FastSyncPivot()
+
+ // Keep a count of uncertain headers to roll back
+ rollback := []*types.Header{}
+ defer func() {
+ if len(rollback) > 0 {
+ // Flatten the headers and roll them back
+ hashes := make([]common.Hash, len(rollback))
+ for i, header := range rollback {
+ hashes[i] = header.Hash()
+ }
+ lh, lfb, lb := d.headHeader().Number, d.headFastBlock().Number(), d.headBlock().Number()
+ d.rollback(hashes)
+ glog.V(logger.Warn).Infof("Rolled back %d headers (LH: %d->%d, FB: %d->%d, LB: %d->%d)",
+ len(hashes), lh, d.headHeader().Number, lfb, d.headFastBlock().Number(), lb, d.headBlock().Number())
+
+ // If we're already past the pivot point, this could be an attack, disable fast sync
+ if rollback[len(rollback)-1].Number.Uint64() > pivot {
+ d.noFast = true
+ }
+ }
+ }()
+
// Create a timeout timer, and the associated hash fetcher
request := time.Now() // time of the last fetch request
timeout := time.NewTimer(0) // timer to dump a non-responsive active peer
@@ -1049,22 +1145,24 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
case <-d.blockCh:
// Out of bounds eth/61 blocks received, ignore them
- case headerPack := <-d.headerCh:
+ case packet := <-d.headerCh:
// Make sure the active peer is giving us the headers
- if headerPack.peerId != p.id {
- glog.V(logger.Debug).Infof("Received headers from incorrect peer (%s)", headerPack.peerId)
+ if packet.PeerId() != p.id {
+ glog.V(logger.Debug).Infof("Received headers from incorrect peer (%s)", packet.PeerId())
break
}
headerReqTimer.UpdateSince(request)
timeout.Stop()
- // If no more headers are inbound, notify the body fetcher and return
- if len(headerPack.headers) == 0 {
+ // If no more headers are inbound, notify the content fetchers and return
+ if packet.Items() == 0 {
glog.V(logger.Debug).Infof("%v: no available headers", p)
- select {
- case d.wakeCh <- false:
- case <-d.cancelCh:
+ for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} {
+ select {
+ case ch <- false:
+ case <-d.cancelCh:
+ }
}
// If no headers were retrieved at all, the peer violated it's TD promise that it had a
// better chain compared to ours. The only exception is if it's promised blocks were
@@ -1081,35 +1179,77 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
if !gotHeaders && td.Cmp(d.getTd(d.headBlock().Hash())) > 0 {
return errStallingPeer
}
+ // If fast or light syncing, ensure promised headers are indeed delivered. This is
+ // needed to detect scenarios where an attacker feeds a bad pivot and then bails out
+ // of delivering the post-pivot blocks that would flag the invalid content.
+ //
+ // This check cannot be executed "as is" for full imports, since blocks may still be
+ // queued for processing when the header download completes. However, as long as the
+ // peer gave us something useful, we're already happy/progressed (above check).
+ if d.mode == FastSync || d.mode == LightSync {
+ if td.Cmp(d.getTd(d.headHeader().Hash())) > 0 {
+ return errStallingPeer
+ }
+ }
+ rollback = nil
return nil
}
gotHeaders = true
+ headers := packet.(*headerPack).headers
// Otherwise insert all the new headers, aborting in case of junk
- glog.V(logger.Detail).Infof("%v: inserting %d headers from #%d", p, len(headerPack.headers), from)
+ glog.V(logger.Detail).Infof("%v: schedule %d headers from #%d", p, len(headers), from)
- inserts := d.queue.Insert(headerPack.headers, from)
- if len(inserts) != len(headerPack.headers) {
- glog.V(logger.Debug).Infof("%v: stale headers", p)
- return errBadPeer
+ if d.mode == FastSync || d.mode == LightSync {
+ // Collect the yet unknown headers to mark them as uncertain
+ unknown := make([]*types.Header, 0, len(headers))
+ for _, header := range headers {
+ if !d.hasHeader(header.Hash()) {
+ unknown = append(unknown, header)
+ }
+ }
+ // If we're importing pure headers, verify based on their recentness
+ frequency := fsHeaderCheckFrequency
+ if headers[len(headers)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot {
+ frequency = 1
+ }
+ if n, err := d.insertHeaders(headers, frequency); err != nil {
+ glog.V(logger.Debug).Infof("%v: invalid header #%d [%x…]: %v", p, headers[n].Number, headers[n].Hash().Bytes()[:4], err)
+ return errInvalidChain
+ }
+ // All verifications passed, store newly found uncertain headers
+ rollback = append(rollback, unknown...)
+ if len(rollback) > fsHeaderSafetyNet {
+ rollback = append(rollback[:0], rollback[len(rollback)-fsHeaderSafetyNet:]...)
+ }
}
- // Notify the block fetcher of new headers, but stop if queue is full
- if d.queue.Pending() < maxQueuedHeaders {
- // We still have headers to fetch, send continuation wake signal (potential)
- select {
- case d.wakeCh <- true:
- default:
+ if d.mode == FullSync || d.mode == FastSync {
+ inserts := d.queue.Schedule(headers, from)
+ if len(inserts) != len(headers) {
+ glog.V(logger.Debug).Infof("%v: stale headers", p)
+ return errBadPeer
}
- } else {
- // Header limit reached, send a termination wake signal (enforced)
- select {
- case d.wakeCh <- false:
- case <-d.cancelCh:
+ }
+ // Notify the content fetchers of new headers, but stop if queue is full
+ cont := d.queue.PendingBlocks() < maxQueuedHeaders || d.queue.PendingReceipts() < maxQueuedHeaders
+ for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} {
+ if cont {
+ // We still have headers to fetch, send continuation wake signal (potential)
+ select {
+ case ch <- true:
+ default:
+ }
+ } else {
+ // Header limit reached, send a termination wake signal (enforced)
+ select {
+ case ch <- false:
+ case <-d.cancelCh:
+ }
+ return nil
}
- return nil
}
// Queue not yet full, fetch the next batch
- from += uint64(len(headerPack.headers))
+ from += uint64(len(headers))
getHeaders(from)
case <-timeout.C:
@@ -1119,9 +1259,11 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
d.dropPeer(p.id)
// Finish the sync gracefully instead of dumping the gathered data though
- select {
- case d.wakeCh <- false:
- case <-d.cancelCh:
+ for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh, d.stateWakeCh} {
+ select {
+ case ch <- false:
+ case <-d.cancelCh:
+ }
}
return nil
}
@@ -1133,22 +1275,119 @@ func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
// and also periodically checking for timeouts.
func (d *Downloader) fetchBodies(from uint64) error {
glog.V(logger.Debug).Infof("Downloading block bodies from #%d", from)
- defer glog.V(logger.Debug).Infof("Block body download terminated")
- // Create a timeout timer for scheduling expiration tasks
+ var (
+ deliver = func(packet dataPack) error {
+ pack := packet.(*bodyPack)
+ return d.queue.DeliverBodies(pack.peerId, pack.transactions, pack.uncles)
+ }
+ expire = func() []string { return d.queue.ExpireBodies(bodyHardTTL) }
+ fetch = func(p *peer, req *fetchRequest) error { return p.FetchBodies(req) }
+ capacity = func(p *peer) int { return p.BlockCapacity() }
+ setIdle = func(p *peer) { p.SetBodiesIdle() }
+ )
+ err := d.fetchParts(errCancelBodyFetch, d.bodyCh, deliver, d.bodyWakeCh, expire,
+ d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ShouldThrottleBlocks, d.queue.ReserveBodies,
+ d.bodyFetchHook, fetch, d.queue.CancelBodies, capacity, d.peers.BodyIdlePeers, setIdle, "Body")
+
+ glog.V(logger.Debug).Infof("Block body download terminated: %v", err)
+ return err
+}
+
+// fetchReceipts iteratively downloads the scheduled block receipts, taking any
+// available peers, reserving a chunk of receipts for each, waiting for delivery
+// and also periodically checking for timeouts.
+func (d *Downloader) fetchReceipts(from uint64) error {
+ glog.V(logger.Debug).Infof("Downloading receipts from #%d", from)
+
+ var (
+ deliver = func(packet dataPack) error {
+ pack := packet.(*receiptPack)
+ return d.queue.DeliverReceipts(pack.peerId, pack.receipts)
+ }
+ expire = func() []string { return d.queue.ExpireReceipts(receiptHardTTL) }
+ fetch = func(p *peer, req *fetchRequest) error { return p.FetchReceipts(req) }
+ capacity = func(p *peer) int { return p.ReceiptCapacity() }
+ setIdle = func(p *peer) { p.SetReceiptsIdle() }
+ )
+ err := d.fetchParts(errCancelReceiptFetch, d.receiptCh, deliver, d.receiptWakeCh, expire,
+ d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ShouldThrottleReceipts, d.queue.ReserveReceipts,
+ d.receiptFetchHook, fetch, d.queue.CancelReceipts, capacity, d.peers.ReceiptIdlePeers, setIdle, "Receipt")
+
+ glog.V(logger.Debug).Infof("Receipt download terminated: %v", err)
+ return err
+}
+
+// fetchNodeData iteratively downloads the scheduled state trie nodes, taking any
+// available peers, reserving a chunk of nodes for each, waiting for delivery and
+// also periodically checking for timeouts.
+func (d *Downloader) fetchNodeData() error {
+ glog.V(logger.Debug).Infof("Downloading node state data")
+
+ var (
+ deliver = func(packet dataPack) error {
+ start := time.Now()
+ return d.queue.DeliverNodeData(packet.PeerId(), packet.(*statePack).states, func(err error, delivered int) {
+ if err != nil {
+ // If the node data processing failed, the root hash is very wrong, abort
+ glog.V(logger.Error).Infof("peer %d: state processing failed: %v", packet.PeerId(), err)
+ d.cancel()
+ return
+ }
+ // Processing succeeded, notify state fetcher and processor of continuation
+ if d.queue.PendingNodeData() == 0 {
+ go d.process()
+ } else {
+ select {
+ case d.stateWakeCh <- true:
+ default:
+ }
+ }
+ // Log a message to the user and return
+ d.syncStatsLock.Lock()
+ defer d.syncStatsLock.Unlock()
+
+ d.syncStatsStateDone += uint64(delivered)
+ glog.V(logger.Info).Infof("imported %d state entries in %v: processed %d in total", delivered, time.Since(start), d.syncStatsStateDone)
+ })
+ }
+ expire = func() []string { return d.queue.ExpireNodeData(stateHardTTL) }
+ throttle = func() bool { return false }
+ reserve = func(p *peer, count int) (*fetchRequest, bool, error) {
+ return d.queue.ReserveNodeData(p, count), false, nil
+ }
+ fetch = func(p *peer, req *fetchRequest) error { return p.FetchNodeData(req) }
+ capacity = func(p *peer) int { return p.NodeDataCapacity() }
+ setIdle = func(p *peer) { p.SetNodeDataIdle() }
+ )
+ err := d.fetchParts(errCancelStateFetch, d.stateCh, deliver, d.stateWakeCh, expire,
+ d.queue.PendingNodeData, d.queue.InFlightNodeData, throttle, reserve, nil, fetch,
+ d.queue.CancelNodeData, capacity, d.peers.NodeDataIdlePeers, setIdle, "State")
+
+ glog.V(logger.Debug).Infof("Node state data download terminated: %v", err)
+ return err
+}
+
+// fetchParts iteratively downloads scheduled block parts, taking any available
+// peers, reserving a chunk of fetch requests for each, waiting for delivery and
+// also periodically checking for timeouts.
+func (d *Downloader) fetchParts(errCancel error, deliveryCh chan dataPack, deliver func(packet dataPack) error, wakeCh chan bool,
+ expire func() []string, pending func() int, inFlight func() bool, throttle func() bool, reserve func(*peer, int) (*fetchRequest, bool, error),
+ fetchHook func([]*types.Header), fetch func(*peer, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peer) int,
+ idle func() ([]*peer, int), setIdle func(*peer), kind string) error {
+
+ // Create a ticker to detect expired retrieval tasks
ticker := time.NewTicker(100 * time.Millisecond)
defer ticker.Stop()
update := make(chan struct{}, 1)
- // Prepare the queue and fetch block bodies until the block header fetcher's done
- d.queue.Prepare(from)
+ // Prepare the queue and fetch block parts until the block header fetcher's done
finished := false
-
for {
select {
case <-d.cancelCh:
- return errCancelBlockFetch
+ return errCancel
case <-d.hashCh:
// Out of bounds eth/61 hashes received, ignore them
@@ -1156,42 +1395,36 @@ func (d *Downloader) fetchBodies(from uint64) error {
case <-d.blockCh:
// Out of bounds eth/61 blocks received, ignore them
- case bodyPack := <-d.bodyCh:
+ case packet := <-deliveryCh:
// If the peer was previously banned and failed to deliver it's pack
// in a reasonable time frame, ignore it's message.
- if peer := d.peers.Peer(bodyPack.peerId); peer != nil {
- // Deliver the received chunk of bodies, and demote in case of errors
- err := d.queue.Deliver(bodyPack.peerId, bodyPack.transactions, bodyPack.uncles)
- switch err {
+ if peer := d.peers.Peer(packet.PeerId()); peer != nil {
+ // Deliver the received chunk of data, and demote in case of errors
+ switch err := deliver(packet); err {
case nil:
- // If no blocks were delivered, demote the peer (need the delivery above)
- if len(bodyPack.transactions) == 0 || len(bodyPack.uncles) == 0 {
+ // If no blocks were delivered, demote the peer (need the delivery above to clean internal queue!)
+ if packet.Items() == 0 {
peer.Demote()
- peer.SetIdle()
- glog.V(logger.Detail).Infof("%s: no block bodies delivered", peer)
+ setIdle(peer)
+ glog.V(logger.Detail).Infof("%s: no %s delivered", peer, strings.ToLower(kind))
break
}
// All was successful, promote the peer and potentially start processing
peer.Promote()
- peer.SetIdle()
- glog.V(logger.Detail).Infof("%s: delivered %d:%d block bodies", peer, len(bodyPack.transactions), len(bodyPack.uncles))
+ setIdle(peer)
+ glog.V(logger.Detail).Infof("%s: delivered %s %s(s)", peer, packet.Stats(), strings.ToLower(kind))
go d.process()
case errInvalidChain:
// The hash chain is invalid (blocks are not ordered properly), abort
return err
- case errInvalidBody:
- // The peer delivered something very bad, drop immediately
- glog.V(logger.Error).Infof("%s: delivered invalid block, dropping", peer)
- d.dropPeer(peer.id)
-
case errNoFetchesPending:
// Peer probably timed out with its delivery but came through
// in the end, demote, but allow to to pull from this peer.
peer.Demote()
- peer.SetIdle()
- glog.V(logger.Detail).Infof("%s: out of bound delivery", peer)
+ setIdle(peer)
+ glog.V(logger.Detail).Infof("%s: out of bound %s delivery", peer, strings.ToLower(kind))
case errStaleDelivery:
// Delivered something completely else than requested, usually
@@ -1199,13 +1432,13 @@ func (d *Downloader) fetchBodies(from uint64) error {
// Don't set it to idle as the original request should still be
// in flight.
peer.Demote()
- glog.V(logger.Detail).Infof("%s: stale delivery", peer)
+ glog.V(logger.Detail).Infof("%s: %s stale delivery", peer, strings.ToLower(kind))
default:
// Peer did something semi-useful, demote but keep it around
peer.Demote()
- peer.SetIdle()
- glog.V(logger.Detail).Infof("%s: delivery partially failed: %v", peer, err)
+ setIdle(peer)
+ glog.V(logger.Detail).Infof("%s: %s delivery partially failed: %v", peer, strings.ToLower(kind), err)
go d.process()
}
}
@@ -1215,7 +1448,7 @@ func (d *Downloader) fetchBodies(from uint64) error {
default:
}
- case cont := <-d.wakeCh:
+ case cont := <-wakeCh:
// The header fetcher sent a continuation flag, check if it's done
if !cont {
finished = true
@@ -1238,65 +1471,80 @@ func (d *Downloader) fetchBodies(from uint64) error {
if d.peers.Len() == 0 {
return errNoPeers
}
- // Check for block body request timeouts and demote the responsible peers
- for _, pid := range d.queue.Expire(bodyHardTTL) {
+ // Check for fetch request timeouts and demote the responsible peers
+ for _, pid := range expire() {
if peer := d.peers.Peer(pid); peer != nil {
peer.Demote()
- glog.V(logger.Detail).Infof("%s: block body delivery timeout", peer)
+ setIdle(peer)
+ glog.V(logger.Detail).Infof("%s: %s delivery timeout", peer, strings.ToLower(kind))
}
}
- // If there's noting more to fetch, wait or terminate
- if d.queue.Pending() == 0 {
- if d.queue.InFlight() == 0 && finished {
- glog.V(logger.Debug).Infof("Block body fetching completed")
+ // If there's nothing more to fetch, wait or terminate
+ if pending() == 0 {
+ if !inFlight() && finished {
+ glog.V(logger.Debug).Infof("%s fetching completed", kind)
return nil
}
break
}
// Send a download request to all idle peers, until throttled
- queuedEmptyBlocks, throttled := false, false
- for _, peer := range d.peers.IdlePeers(eth62) {
+ progressed, throttled, running := false, false, inFlight()
+ idles, total := idle()
+
+ for _, peer := range idles {
// Short circuit if throttling activated
- if d.queue.Throttle() {
+ if throttle() {
throttled = true
break
}
- // Reserve a chunk of hashes for a peer. A nil can mean either that
- // no more hashes are available, or that the peer is known not to
+ // Reserve a chunk of fetches for a peer. A nil can mean either that
+ // no more headers are available, or that the peer is known not to
// have them.
- request, process, err := d.queue.Reserve(peer, peer.Capacity())
+ request, progress, err := reserve(peer, capacity(peer))
if err != nil {
return err
}
- if process {
- queuedEmptyBlocks = true
+ if progress {
+ progressed = true
go d.process()
}
if request == nil {
continue
}
if glog.V(logger.Detail) {
- glog.Infof("%s: requesting %d block bodies", peer, len(request.Headers))
+ if len(request.Headers) > 0 {
+ glog.Infof("%s: requesting %d %s(s), first at #%d", peer, len(request.Headers), strings.ToLower(kind), request.Headers[0].Number)
+ } else {
+ glog.Infof("%s: requesting %d %s(s)", peer, len(request.Hashes), strings.ToLower(kind))
+ }
}
// Fetch the chunk and make sure any errors return the hashes to the queue
- if d.bodyFetchHook != nil {
- d.bodyFetchHook(request.Headers)
+ if fetchHook != nil {
+ fetchHook(request.Headers)
}
- if err := peer.Fetch(request); err != nil {
- glog.V(logger.Error).Infof("%v: fetch failed, rescheduling", peer)
- d.queue.Cancel(request)
+ if err := fetch(peer, request); err != nil {
+ // Although we could try and make an attempt to fix this, this error really
+ // means that we've double allocated a fetch task to a peer. If that is the
+ // case, the internal state of the downloader and the queue is very wrong so
+ // better hard crash and note the error instead of silently accumulating into
+ // a much bigger issue.
+ panic(fmt.Sprintf("%v: %s fetch assignment failed, hard panic", peer, strings.ToLower(kind)))
+ cancel(request) // noop for now
}
+ running = true
}
// Make sure that we have peers available for fetching. If all peers have been tried
// and all failed throw an error
- if !queuedEmptyBlocks && !throttled && d.queue.InFlight() == 0 {
+ if !progressed && !throttled && !running && len(idles) == total && pending() > 0 {
return errPeersUnavailable
}
}
}
}
-// process takes blocks from the queue and tries to import them into the chain.
+// process takes fetch results from the queue and tries to import them into the
+// chain. The type of import operation will depend on the result contents:
+// -
//
// The algorithmic flow is as follows:
// - The `processing` flag is swapped to 1 to ensure singleton access
@@ -1317,10 +1565,10 @@ func (d *Downloader) process() {
}
// If the processor just exited, but there are freshly pending items, try to
// reenter. This is needed because the goroutine spinned up for processing
- // the fresh blocks might have been rejected entry to to this present thread
+ // the fresh results might have been rejected entry to to this present thread
// not yet releasing the `processing` state.
defer func() {
- if atomic.LoadInt32(&d.interrupt) == 0 && d.queue.GetHeadBlock() != nil {
+ if atomic.LoadInt32(&d.interrupt) == 0 && d.queue.GetHeadResult() != nil {
d.process()
}
}()
@@ -1328,134 +1576,111 @@ func (d *Downloader) process() {
// the import statistics to zero.
defer atomic.StoreInt32(&d.processing, 0)
- // Repeat the processing as long as there are blocks to import
+ // Repeat the processing as long as there are results to process
for {
- // Fetch the next batch of blocks
- blocks := d.queue.TakeBlocks()
- if len(blocks) == 0 {
+ // Fetch the next batch of results
+ pivot := d.queue.FastSyncPivot() // Fetch pivot before results to prevent reset race
+ results := d.queue.TakeResults()
+ if len(results) == 0 {
return
}
if d.chainInsertHook != nil {
- d.chainInsertHook(blocks)
+ d.chainInsertHook(results)
}
// Actually import the blocks
- glog.V(logger.Debug).Infof("Inserting chain with %d blocks (#%v - #%v)\n", len(blocks), blocks[0].RawBlock.Number(), blocks[len(blocks)-1].RawBlock.Number())
- for len(blocks) != 0 {
+ if glog.V(logger.Debug) {
+ first, last := results[0].Header, results[len(results)-1].Header
+ glog.Infof("Inserting chain with %d items (#%d [%x…] - #%d [%x…])", len(results), first.Number, first.Hash().Bytes()[:4], last.Number, last.Hash().Bytes()[:4])
+ }
+ for len(results) != 0 {
// Check for any termination requests
if atomic.LoadInt32(&d.interrupt) == 1 {
return
}
- // Retrieve the first batch of blocks to insert
- max := int(math.Min(float64(len(blocks)), float64(maxBlockProcess)))
- raw := make(types.Blocks, 0, max)
- for _, block := range blocks[:max] {
- raw = append(raw, block.RawBlock)
+ // Retrieve the a batch of results to import
+ var (
+ blocks = make([]*types.Block, 0, maxResultsProcess)
+ receipts = make([]types.Receipts, 0, maxResultsProcess)
+ )
+ items := int(math.Min(float64(len(results)), float64(maxResultsProcess)))
+ for _, result := range results[:items] {
+ switch {
+ case d.mode == FullSync:
+ blocks = append(blocks, types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles))
+ case d.mode == FastSync:
+ blocks = append(blocks, types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles))
+ if result.Header.Number.Uint64() <= pivot {
+ receipts = append(receipts, result.Receipts)
+ }
+ }
+ }
+ // Try to process the results, aborting if there's an error
+ var (
+ err error
+ index int
+ )
+ switch {
+ case len(receipts) > 0:
+ index, err = d.insertReceipts(blocks, receipts)
+ if err == nil && blocks[len(blocks)-1].NumberU64() == pivot {
+ glog.V(logger.Debug).Infof("Committing block #%d [%x…] as the new head", blocks[len(blocks)-1].Number(), blocks[len(blocks)-1].Hash().Bytes()[:4])
+ index, err = len(blocks)-1, d.commitHeadBlock(blocks[len(blocks)-1].Hash())
+ }
+ default:
+ index, err = d.insertBlocks(blocks)
}
- // Try to inset the blocks, drop the originating peer if there's an error
- index, err := d.insertChain(raw)
if err != nil {
- glog.V(logger.Debug).Infof("Block #%d import failed: %v", raw[index].NumberU64(), err)
- d.dropPeer(blocks[index].OriginPeer)
+ glog.V(logger.Debug).Infof("Result #%d [%x…] processing failed: %v", results[index].Header.Number, results[index].Header.Hash().Bytes()[:4], err)
d.cancel()
return
}
- blocks = blocks[max:]
+ // Shift the results to the next batch
+ results = results[items:]
}
}
}
-// DeliverHashes61 injects a new batch of hashes received from a remote node into
+// DeliverHashes injects a new batch of hashes received from a remote node into
// the download schedule. This is usually invoked through the BlockHashesMsg by
// the protocol handler.
-func (d *Downloader) DeliverHashes61(id string, hashes []common.Hash) (err error) {
- // Update the delivery metrics for both good and failed deliveries
- hashInMeter.Mark(int64(len(hashes)))
- defer func() {
- if err != nil {
- hashDropMeter.Mark(int64(len(hashes)))
- }
- }()
- // Make sure the downloader is active
- if atomic.LoadInt32(&d.synchronising) == 0 {
- return errNoSyncActive
- }
- // Deliver or abort if the sync is canceled while queuing
- d.cancelLock.RLock()
- cancel := d.cancelCh
- d.cancelLock.RUnlock()
-
- select {
- case d.hashCh <- hashPack{id, hashes}:
- return nil
-
- case <-cancel:
- return errNoSyncActive
- }
+func (d *Downloader) DeliverHashes(id string, hashes []common.Hash) (err error) {
+ return d.deliver(id, d.hashCh, &hashPack{id, hashes}, hashInMeter, hashDropMeter)
}
-// DeliverBlocks61 injects a new batch of blocks received from a remote node.
+// DeliverBlocks injects a new batch of blocks received from a remote node.
// This is usually invoked through the BlocksMsg by the protocol handler.
-func (d *Downloader) DeliverBlocks61(id string, blocks []*types.Block) (err error) {
- // Update the delivery metrics for both good and failed deliveries
- blockInMeter.Mark(int64(len(blocks)))
- defer func() {
- if err != nil {
- blockDropMeter.Mark(int64(len(blocks)))
- }
- }()
- // Make sure the downloader is active
- if atomic.LoadInt32(&d.synchronising) == 0 {
- return errNoSyncActive
- }
- // Deliver or abort if the sync is canceled while queuing
- d.cancelLock.RLock()
- cancel := d.cancelCh
- d.cancelLock.RUnlock()
-
- select {
- case d.blockCh <- blockPack{id, blocks}:
- return nil
-
- case <-cancel:
- return errNoSyncActive
- }
+func (d *Downloader) DeliverBlocks(id string, blocks []*types.Block) (err error) {
+ return d.deliver(id, d.blockCh, &blockPack{id, blocks}, blockInMeter, blockDropMeter)
}
// DeliverHeaders injects a new batch of blck headers received from a remote
// node into the download schedule.
func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) (err error) {
- // Update the delivery metrics for both good and failed deliveries
- headerInMeter.Mark(int64(len(headers)))
- defer func() {
- if err != nil {
- headerDropMeter.Mark(int64(len(headers)))
- }
- }()
- // Make sure the downloader is active
- if atomic.LoadInt32(&d.synchronising) == 0 {
- return errNoSyncActive
- }
- // Deliver or abort if the sync is canceled while queuing
- d.cancelLock.RLock()
- cancel := d.cancelCh
- d.cancelLock.RUnlock()
-
- select {
- case d.headerCh <- headerPack{id, headers}:
- return nil
-
- case <-cancel:
- return errNoSyncActive
- }
+ return d.deliver(id, d.headerCh, &headerPack{id, headers}, headerInMeter, headerDropMeter)
}
// DeliverBodies injects a new batch of block bodies received from a remote node.
func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) (err error) {
+ return d.deliver(id, d.bodyCh, &bodyPack{id, transactions, uncles}, bodyInMeter, bodyDropMeter)
+}
+
+// DeliverReceipts injects a new batch of receipts received from a remote node.
+func (d *Downloader) DeliverReceipts(id string, receipts [][]*types.Receipt) (err error) {
+ return d.deliver(id, d.receiptCh, &receiptPack{id, receipts}, receiptInMeter, receiptDropMeter)
+}
+
+// DeliverNodeData injects a new batch of node state data received from a remote node.
+func (d *Downloader) DeliverNodeData(id string, data [][]byte) (err error) {
+ return d.deliver(id, d.stateCh, &statePack{id, data}, stateInMeter, stateDropMeter)
+}
+
+// deliver injects a new batch of data received from a remote node.
+func (d *Downloader) deliver(id string, destCh chan dataPack, packet dataPack, inMeter, dropMeter metrics.Meter) (err error) {
// Update the delivery metrics for both good and failed deliveries
- bodyInMeter.Mark(int64(len(transactions)))
+ inMeter.Mark(int64(packet.Items()))
defer func() {
if err != nil {
- bodyDropMeter.Mark(int64(len(transactions)))
+ dropMeter.Mark(int64(packet.Items()))
}
}()
// Make sure the downloader is active
@@ -1468,7 +1693,7 @@ func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transactio
d.cancelLock.RUnlock()
select {
- case d.bodyCh <- bodyPack{id, transactions, uncles}:
+ case destCh <- packet:
return nil
case <-cancel:
diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go
index 96096527e..ef6f74a6b 100644
--- a/eth/downloader/downloader_test.go
+++ b/eth/downloader/downloader_test.go
@@ -27,11 +27,13 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core"
+ "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/event"
"github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/trie"
)
var (
@@ -45,8 +47,9 @@ var (
// the returned hash chain is ordered head->parent. In addition, every 3rd block
// contains a transaction and every 5th an uncle to allow testing correct block
// reassembly.
-func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
- blocks := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
+func makeChain(n int, seed byte, parent *types.Block, parentReceipts types.Receipts) ([]common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]types.Receipts) {
+ // Generate the block chain
+ blocks, receipts := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
block.SetCoinbase(common.Address{seed})
// If the block number is multiple of 3, send a bonus transaction to the miner
@@ -62,44 +65,72 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common
block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))})
}
})
+ // Convert the block-chain into a hash-chain and header/block maps
hashes := make([]common.Hash, n+1)
hashes[len(hashes)-1] = parent.Hash()
+
+ headerm := make(map[common.Hash]*types.Header, n+1)
+ headerm[parent.Hash()] = parent.Header()
+
blockm := make(map[common.Hash]*types.Block, n+1)
blockm[parent.Hash()] = parent
+
+ receiptm := make(map[common.Hash]types.Receipts, n+1)
+ receiptm[parent.Hash()] = parentReceipts
+
for i, b := range blocks {
hashes[len(hashes)-i-2] = b.Hash()
+ headerm[b.Hash()] = b.Header()
blockm[b.Hash()] = b
+ receiptm[b.Hash()] = receipts[i]
}
- return hashes, blockm
+ return hashes, headerm, blockm, receiptm
}
// makeChainFork creates two chains of length n, such that h1[:f] and
// h2[:f] are different but have a common suffix of length n-f.
-func makeChainFork(n, f int, parent *types.Block) (h1, h2 []common.Hash, b1, b2 map[common.Hash]*types.Block) {
- // Create the common suffix.
- h, b := makeChain(n-f, 0, parent)
- // Create the forks.
- h1, b1 = makeChain(f, 1, b[h[0]])
- h1 = append(h1, h[1:]...)
- h2, b2 = makeChain(f, 2, b[h[0]])
- h2 = append(h2, h[1:]...)
- for hash, block := range b {
- b1[hash] = block
- b2[hash] = block
+func makeChainFork(n, f int, parent *types.Block, parentReceipts types.Receipts) ([]common.Hash, []common.Hash, map[common.Hash]*types.Header, map[common.Hash]*types.Header, map[common.Hash]*types.Block, map[common.Hash]*types.Block, map[common.Hash]types.Receipts, map[common.Hash]types.Receipts) {
+ // Create the common suffix
+ hashes, headers, blocks, receipts := makeChain(n-f, 0, parent, parentReceipts)
+
+ // Create the forks
+ hashes1, headers1, blocks1, receipts1 := makeChain(f, 1, blocks[hashes[0]], receipts[hashes[0]])
+ hashes1 = append(hashes1, hashes[1:]...)
+
+ hashes2, headers2, blocks2, receipts2 := makeChain(f, 2, blocks[hashes[0]], receipts[hashes[0]])
+ hashes2 = append(hashes2, hashes[1:]...)
+
+ for hash, header := range headers {
+ headers1[hash] = header
+ headers2[hash] = header
}
- return h1, h2, b1, b2
+ for hash, block := range blocks {
+ blocks1[hash] = block
+ blocks2[hash] = block
+ }
+ for hash, receipt := range receipts {
+ receipts1[hash] = receipt
+ receipts2[hash] = receipt
+ }
+ return hashes1, hashes2, headers1, headers2, blocks1, blocks2, receipts1, receipts2
}
// downloadTester is a test simulator for mocking out local block chain.
type downloadTester struct {
+ stateDb ethdb.Database
downloader *Downloader
- ownHashes []common.Hash // Hash chain belonging to the tester
- ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester
- ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain
- peerHashes map[string][]common.Hash // Hash chain belonging to different test peers
- peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers
- peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains
+ ownHashes []common.Hash // Hash chain belonging to the tester
+ ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester
+ ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester
+ ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester
+ ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain
+
+ peerHashes map[string][]common.Hash // Hash chain belonging to different test peers
+ peerHeaders map[string]map[common.Hash]*types.Header // Headers belonging to different test peers
+ peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers
+ peerReceipts map[string]map[common.Hash]types.Receipts // Receipts belonging to different test peers
+ peerChainTds map[string]map[common.Hash]*big.Int // Total difficulties of the blocks in the peer chains
lock sync.RWMutex
}
@@ -108,19 +139,26 @@ type downloadTester struct {
func newTester() *downloadTester {
tester := &downloadTester{
ownHashes: []common.Hash{genesis.Hash()},
+ ownHeaders: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()},
ownBlocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
+ ownReceipts: map[common.Hash]types.Receipts{genesis.Hash(): nil},
ownChainTd: map[common.Hash]*big.Int{genesis.Hash(): genesis.Difficulty()},
peerHashes: make(map[string][]common.Hash),
+ peerHeaders: make(map[string]map[common.Hash]*types.Header),
peerBlocks: make(map[string]map[common.Hash]*types.Block),
+ peerReceipts: make(map[string]map[common.Hash]types.Receipts),
peerChainTds: make(map[string]map[common.Hash]*big.Int),
}
- tester.downloader = New(new(event.TypeMux), tester.hasBlock, tester.getBlock, tester.headBlock, tester.getTd, tester.insertChain, tester.dropPeer)
+ tester.stateDb, _ = ethdb.NewMemDatabase()
+ tester.downloader = New(tester.stateDb, new(event.TypeMux), tester.hasHeader, tester.hasBlock, tester.getHeader,
+ tester.getBlock, tester.headHeader, tester.headBlock, tester.headFastBlock, tester.commitHeadBlock, tester.getTd,
+ tester.insertHeaders, tester.insertBlocks, tester.insertReceipts, tester.rollback, tester.dropPeer)
return tester
}
// sync starts synchronizing with a remote peer, blocking until it completes.
-func (dl *downloadTester) sync(id string, td *big.Int) error {
+func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error {
dl.lock.RLock()
hash := dl.peerHashes[id][0]
// If no particular TD was requested, load from the peer's blockchain
@@ -132,11 +170,10 @@ func (dl *downloadTester) sync(id string, td *big.Int) error {
}
dl.lock.RUnlock()
- err := dl.downloader.synchronise(id, hash, td)
+ err := dl.downloader.synchronise(id, hash, td, mode)
for {
// If the queue is empty and processing stopped, break
- hashes, blocks := dl.downloader.queue.Size()
- if hashes+blocks == 0 && atomic.LoadInt32(&dl.downloader.processing) == 0 {
+ if dl.downloader.queue.Idle() && atomic.LoadInt32(&dl.downloader.processing) == 0 {
break
}
// Otherwise sleep a bit and retry
@@ -145,12 +182,22 @@ func (dl *downloadTester) sync(id string, td *big.Int) error {
return err
}
-// hasBlock checks if a block is pres ent in the testers canonical chain.
+// hasHeader checks if a header is present in the testers canonical chain.
+func (dl *downloadTester) hasHeader(hash common.Hash) bool {
+ return dl.getHeader(hash) != nil
+}
+
+// hasBlock checks if a block is present in the testers canonical chain.
func (dl *downloadTester) hasBlock(hash common.Hash) bool {
+ return dl.getBlock(hash) != nil
+}
+
+// getHeader retrieves a header from the testers canonical chain.
+func (dl *downloadTester) getHeader(hash common.Hash) *types.Header {
dl.lock.RLock()
defer dl.lock.RUnlock()
- return dl.getBlock(hash) != nil
+ return dl.ownHeaders[hash]
}
// getBlock retrieves a block from the testers canonical chain.
@@ -161,12 +208,55 @@ func (dl *downloadTester) getBlock(hash common.Hash) *types.Block {
return dl.ownBlocks[hash]
}
+// headHeader retrieves the current head header from the canonical chain.
+func (dl *downloadTester) headHeader() *types.Header {
+ dl.lock.RLock()
+ defer dl.lock.RUnlock()
+
+ for i := len(dl.ownHashes) - 1; i >= 0; i-- {
+ if header := dl.ownHeaders[dl.ownHashes[i]]; header != nil {
+ return header
+ }
+ }
+ return genesis.Header()
+}
+
// headBlock retrieves the current head block from the canonical chain.
func (dl *downloadTester) headBlock() *types.Block {
dl.lock.RLock()
defer dl.lock.RUnlock()
- return dl.getBlock(dl.ownHashes[len(dl.ownHashes)-1])
+ for i := len(dl.ownHashes) - 1; i >= 0; i-- {
+ if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil {
+ if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil {
+ return block
+ }
+ }
+ }
+ return genesis
+}
+
+// headFastBlock retrieves the current head fast-sync block from the canonical chain.
+func (dl *downloadTester) headFastBlock() *types.Block {
+ dl.lock.RLock()
+ defer dl.lock.RUnlock()
+
+ for i := len(dl.ownHashes) - 1; i >= 0; i-- {
+ if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil {
+ return block
+ }
+ }
+ return genesis
+}
+
+// commitHeadBlock manually sets the head block to a given hash.
+func (dl *downloadTester) commitHeadBlock(hash common.Hash) error {
+ // For now only check that the state trie is correct
+ if block := dl.getBlock(hash); block != nil {
+ _, err := trie.NewSecure(block.Root(), dl.stateDb)
+ return err
+ }
+ return fmt.Errorf("non existent block: %x", hash[:4])
}
// getTd retrieves the block's total difficulty from the canonical chain.
@@ -177,8 +267,37 @@ func (dl *downloadTester) getTd(hash common.Hash) *big.Int {
return dl.ownChainTd[hash]
}
-// insertChain injects a new batch of blocks into the simulated chain.
-func (dl *downloadTester) insertChain(blocks types.Blocks) (int, error) {
+// insertHeaders injects a new batch of headers into the simulated chain.
+func (dl *downloadTester) insertHeaders(headers []*types.Header, checkFreq int) (int, error) {
+ dl.lock.Lock()
+ defer dl.lock.Unlock()
+
+ // Do a quick check, as the blockchain.InsertHeaderChain doesn't insert anthing in case of errors
+ if _, ok := dl.ownHeaders[headers[0].ParentHash]; !ok {
+ return 0, errors.New("unknown parent")
+ }
+ for i := 1; i < len(headers); i++ {
+ if headers[i].ParentHash != headers[i-1].Hash() {
+ return i, errors.New("unknown parent")
+ }
+ }
+ // Do a full insert if pre-checks passed
+ for i, header := range headers {
+ if _, ok := dl.ownHeaders[header.Hash()]; ok {
+ continue
+ }
+ if _, ok := dl.ownHeaders[header.ParentHash]; !ok {
+ return i, errors.New("unknown parent")
+ }
+ dl.ownHashes = append(dl.ownHashes, header.Hash())
+ dl.ownHeaders[header.Hash()] = header
+ dl.ownChainTd[header.Hash()] = new(big.Int).Add(dl.ownChainTd[header.ParentHash], header.Difficulty)
+ }
+ return len(headers), nil
+}
+
+// insertBlocks injects a new batch of blocks into the simulated chain.
+func (dl *downloadTester) insertBlocks(blocks types.Blocks) (int, error) {
dl.lock.Lock()
defer dl.lock.Unlock()
@@ -186,50 +305,112 @@ func (dl *downloadTester) insertChain(blocks types.Blocks) (int, error) {
if _, ok := dl.ownBlocks[block.ParentHash()]; !ok {
return i, errors.New("unknown parent")
}
- dl.ownHashes = append(dl.ownHashes, block.Hash())
+ if _, ok := dl.ownHeaders[block.Hash()]; !ok {
+ dl.ownHashes = append(dl.ownHashes, block.Hash())
+ dl.ownHeaders[block.Hash()] = block.Header()
+ }
dl.ownBlocks[block.Hash()] = block
- dl.ownChainTd[block.Hash()] = dl.ownChainTd[block.ParentHash()]
+ dl.stateDb.Put(block.Root().Bytes(), []byte{0x00})
+ dl.ownChainTd[block.Hash()] = new(big.Int).Add(dl.ownChainTd[block.ParentHash()], block.Difficulty())
}
return len(blocks), nil
}
+// insertReceipts injects a new batch of blocks into the simulated chain.
+func (dl *downloadTester) insertReceipts(blocks types.Blocks, receipts []types.Receipts) (int, error) {
+ dl.lock.Lock()
+ defer dl.lock.Unlock()
+
+ for i := 0; i < len(blocks) && i < len(receipts); i++ {
+ if _, ok := dl.ownHeaders[blocks[i].Hash()]; !ok {
+ return i, errors.New("unknown owner")
+ }
+ if _, ok := dl.ownBlocks[blocks[i].ParentHash()]; !ok {
+ return i, errors.New("unknown parent")
+ }
+ dl.ownBlocks[blocks[i].Hash()] = blocks[i]
+ dl.ownReceipts[blocks[i].Hash()] = receipts[i]
+ }
+ return len(blocks), nil
+}
+
+// rollback removes some recently added elements from the chain.
+func (dl *downloadTester) rollback(hashes []common.Hash) {
+ dl.lock.Lock()
+ defer dl.lock.Unlock()
+
+ for i := len(hashes) - 1; i >= 0; i-- {
+ if dl.ownHashes[len(dl.ownHashes)-1] == hashes[i] {
+ dl.ownHashes = dl.ownHashes[:len(dl.ownHashes)-1]
+ }
+ delete(dl.ownChainTd, hashes[i])
+ delete(dl.ownHeaders, hashes[i])
+ delete(dl.ownReceipts, hashes[i])
+ delete(dl.ownBlocks, hashes[i])
+ }
+}
+
// newPeer registers a new block download source into the downloader.
-func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, blocks map[common.Hash]*types.Block) error {
- return dl.newSlowPeer(id, version, hashes, blocks, 0)
+func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts) error {
+ return dl.newSlowPeer(id, version, hashes, headers, blocks, receipts, 0)
}
// newSlowPeer registers a new block download source into the downloader, with a
// specific delay time on processing the network packets sent to it, simulating
// potentially slow network IO.
-func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, blocks map[common.Hash]*types.Block, delay time.Duration) error {
+func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, headers map[common.Hash]*types.Header, blocks map[common.Hash]*types.Block, receipts map[common.Hash]types.Receipts, delay time.Duration) error {
dl.lock.Lock()
defer dl.lock.Unlock()
var err error
switch version {
case 61:
- err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay), nil, nil, nil)
+ err = dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay), nil, nil, nil, nil, nil)
case 62:
- err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
+ err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), nil, nil)
case 63:
- err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
+ err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay))
case 64:
- err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
+ err = dl.downloader.RegisterPeer(id, version, hashes[0], nil, nil, nil, dl.peerGetRelHeadersFn(id, delay), dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay), dl.peerGetReceiptsFn(id, delay), dl.peerGetNodeDataFn(id, delay))
}
if err == nil {
- // Assign the owned hashes and blocks to the peer (deep copy)
+ // Assign the owned hashes, headers and blocks to the peer (deep copy)
dl.peerHashes[id] = make([]common.Hash, len(hashes))
copy(dl.peerHashes[id], hashes)
+ dl.peerHeaders[id] = make(map[common.Hash]*types.Header)
dl.peerBlocks[id] = make(map[common.Hash]*types.Block)
+ dl.peerReceipts[id] = make(map[common.Hash]types.Receipts)
dl.peerChainTds[id] = make(map[common.Hash]*big.Int)
- for _, hash := range hashes {
+
+ genesis := hashes[len(hashes)-1]
+ if header := headers[genesis]; header != nil {
+ dl.peerHeaders[id][genesis] = header
+ dl.peerChainTds[id][genesis] = header.Difficulty
+ }
+ if block := blocks[genesis]; block != nil {
+ dl.peerBlocks[id][genesis] = block
+ dl.peerChainTds[id][genesis] = block.Difficulty()
+ }
+
+ for i := len(hashes) - 2; i >= 0; i-- {
+ hash := hashes[i]
+
+ if header, ok := headers[hash]; ok {
+ dl.peerHeaders[id][hash] = header
+ if _, ok := dl.peerHeaders[id][header.ParentHash]; ok {
+ dl.peerChainTds[id][hash] = new(big.Int).Add(header.Difficulty, dl.peerChainTds[id][header.ParentHash])
+ }
+ }
if block, ok := blocks[hash]; ok {
dl.peerBlocks[id][hash] = block
- if parent, ok := dl.peerBlocks[id][block.ParentHash()]; ok {
- dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][parent.Hash()])
+ if _, ok := dl.peerBlocks[id][block.ParentHash()]; ok {
+ dl.peerChainTds[id][hash] = new(big.Int).Add(block.Difficulty(), dl.peerChainTds[id][block.ParentHash()])
}
}
+ if receipt, ok := receipts[hash]; ok {
+ dl.peerReceipts[id][hash] = receipt
+ }
}
}
return err
@@ -241,6 +422,7 @@ func (dl *downloadTester) dropPeer(id string) {
defer dl.lock.Unlock()
delete(dl.peerHashes, id)
+ delete(dl.peerHeaders, id)
delete(dl.peerBlocks, id)
delete(dl.peerChainTds, id)
@@ -273,7 +455,7 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun
// Delay delivery a bit to allow attacks to unfold
go func() {
time.Sleep(time.Millisecond)
- dl.downloader.DeliverHashes61(id, result)
+ dl.downloader.DeliverHashes(id, result)
}()
return nil
}
@@ -298,7 +480,7 @@ func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) fun
// Delay delivery a bit to allow attacks to unfold
go func() {
time.Sleep(time.Millisecond)
- dl.downloader.DeliverHashes61(id, result)
+ dl.downloader.DeliverHashes(id, result)
}()
return nil
}
@@ -321,7 +503,7 @@ func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([
result = append(result, block)
}
}
- go dl.downloader.DeliverBlocks61(id, result)
+ go dl.downloader.DeliverBlocks(id, result)
return nil
}
@@ -358,13 +540,13 @@ func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) fu
dl.lock.RLock()
defer dl.lock.RUnlock()
- // Gather the next batch of hashes
+ // Gather the next batch of headers
hashes := dl.peerHashes[id]
- blocks := dl.peerBlocks[id]
+ headers := dl.peerHeaders[id]
result := make([]*types.Header, 0, amount)
for i := 0; i < amount && len(hashes)-int(origin)-1-i >= 0; i++ {
- if block, ok := blocks[hashes[len(hashes)-int(origin)-1-i]]; ok {
- result = append(result, block.Header())
+ if header, ok := headers[hashes[len(hashes)-int(origin)-1-i]]; ok {
+ result = append(result, header)
}
}
// Delay delivery a bit to allow attacks to unfold
@@ -403,56 +585,163 @@ func (dl *downloadTester) peerGetBodiesFn(id string, delay time.Duration) func([
}
}
+// peerGetReceiptsFn constructs a getReceipts method associated with a particular
+// peer in the download tester. The returned function can be used to retrieve
+// batches of block receipts from the particularly requested peer.
+func (dl *downloadTester) peerGetReceiptsFn(id string, delay time.Duration) func([]common.Hash) error {
+ return func(hashes []common.Hash) error {
+ time.Sleep(delay)
+
+ dl.lock.RLock()
+ defer dl.lock.RUnlock()
+
+ receipts := dl.peerReceipts[id]
+
+ results := make([][]*types.Receipt, 0, len(hashes))
+ for _, hash := range hashes {
+ if receipt, ok := receipts[hash]; ok {
+ results = append(results, receipt)
+ }
+ }
+ go dl.downloader.DeliverReceipts(id, results)
+
+ return nil
+ }
+}
+
+// peerGetNodeDataFn constructs a getNodeData method associated with a particular
+// peer in the download tester. The returned function can be used to retrieve
+// batches of node state data from the particularly requested peer.
+func (dl *downloadTester) peerGetNodeDataFn(id string, delay time.Duration) func([]common.Hash) error {
+ return func(hashes []common.Hash) error {
+ time.Sleep(delay)
+
+ dl.lock.RLock()
+ defer dl.lock.RUnlock()
+
+ results := make([][]byte, 0, len(hashes))
+ for _, hash := range hashes {
+ if data, err := testdb.Get(hash.Bytes()); err == nil {
+ results = append(results, data)
+ }
+ }
+ go dl.downloader.DeliverNodeData(id, results)
+
+ return nil
+ }
+}
+
+// assertOwnChain checks if the local chain contains the correct number of items
+// of the various chain components.
+func assertOwnChain(t *testing.T, tester *downloadTester, length int) {
+ assertOwnForkedChain(t, tester, 1, []int{length})
+}
+
+// assertOwnForkedChain checks if the local forked chain contains the correct
+// number of items of the various chain components.
+func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) {
+ // Initialize the counters for the first fork
+ headers, blocks := lengths[0], lengths[0]
+
+ minReceipts, maxReceipts := lengths[0]-fsMinFullBlocks-fsPivotInterval, lengths[0]-fsMinFullBlocks
+ if minReceipts < 0 {
+ minReceipts = 1
+ }
+ if maxReceipts < 0 {
+ maxReceipts = 1
+ }
+ // Update the counters for each subsequent fork
+ for _, length := range lengths[1:] {
+ headers += length - common
+ blocks += length - common
+
+ minReceipts += length - common - fsMinFullBlocks - fsPivotInterval
+ maxReceipts += length - common - fsMinFullBlocks
+ }
+ switch tester.downloader.mode {
+ case FullSync:
+ minReceipts, maxReceipts = 1, 1
+ case LightSync:
+ blocks, minReceipts, maxReceipts = 1, 1, 1
+ }
+ if hs := len(tester.ownHeaders); hs != headers {
+ t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers)
+ }
+ if bs := len(tester.ownBlocks); bs != blocks {
+ t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks)
+ }
+ if rs := len(tester.ownReceipts); rs < minReceipts || rs > maxReceipts {
+ t.Fatalf("synchronised receipts mismatch: have %v, want between [%v, %v]", rs, minReceipts, maxReceipts)
+ }
+ // Verify the state trie too for fast syncs
+ if tester.downloader.mode == FastSync {
+ index := 0
+ if pivot := int(tester.downloader.queue.fastSyncPivot); pivot < common {
+ index = pivot
+ } else {
+ index = len(tester.ownHashes) - lengths[len(lengths)-1] + int(tester.downloader.queue.fastSyncPivot)
+ }
+ if index > 0 {
+ if statedb, err := state.New(tester.ownHeaders[tester.ownHashes[index]].Root, tester.stateDb); statedb == nil || err != nil {
+ t.Fatalf("state reconstruction failed: %v", err)
+ }
+ }
+ }
+}
+
// Tests that simple synchronization against a canonical chain works correctly.
// In this test common ancestor lookup should be short circuited and not require
// binary searching.
-func TestCanonicalSynchronisation61(t *testing.T) { testCanonicalSynchronisation(t, 61) }
-func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62) }
-func TestCanonicalSynchronisation63(t *testing.T) { testCanonicalSynchronisation(t, 63) }
-func TestCanonicalSynchronisation64(t *testing.T) { testCanonicalSynchronisation(t, 64) }
+func TestCanonicalSynchronisation61(t *testing.T) { testCanonicalSynchronisation(t, 61, FullSync) }
+func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62, FullSync) }
+func TestCanonicalSynchronisation63Full(t *testing.T) { testCanonicalSynchronisation(t, 63, FullSync) }
+func TestCanonicalSynchronisation63Fast(t *testing.T) { testCanonicalSynchronisation(t, 63, FastSync) }
+func TestCanonicalSynchronisation64Full(t *testing.T) { testCanonicalSynchronisation(t, 64, FullSync) }
+func TestCanonicalSynchronisation64Fast(t *testing.T) { testCanonicalSynchronisation(t, 64, FastSync) }
+func TestCanonicalSynchronisation64Light(t *testing.T) { testCanonicalSynchronisation(t, 64, LightSync) }
-func testCanonicalSynchronisation(t *testing.T, protocol int) {
+func testCanonicalSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
- tester.newPeer("peer", protocol, hashes, blocks)
+ tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
- // Synchronise with the peer and make sure all blocks were retrieved
- if err := tester.sync("peer", nil); err != nil {
+ // Synchronise with the peer and make sure all relevant data was retrieved
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != targetBlocks+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, targetBlocks+1)
- }
+ assertOwnChain(t, tester, targetBlocks+1)
}
// Tests that if a large batch of blocks are being downloaded, it is throttled
// until the cached blocks are retrieved.
-func TestThrottling61(t *testing.T) { testThrottling(t, 61) }
-func TestThrottling62(t *testing.T) { testThrottling(t, 62) }
-func TestThrottling63(t *testing.T) { testThrottling(t, 63) }
-func TestThrottling64(t *testing.T) { testThrottling(t, 64) }
+func TestThrottling61(t *testing.T) { testThrottling(t, 61, FullSync) }
+func TestThrottling62(t *testing.T) { testThrottling(t, 62, FullSync) }
+func TestThrottling63Full(t *testing.T) { testThrottling(t, 63, FullSync) }
+func TestThrottling63Fast(t *testing.T) { testThrottling(t, 63, FastSync) }
+func TestThrottling64Full(t *testing.T) { testThrottling(t, 64, FullSync) }
+func TestThrottling64Fast(t *testing.T) { testThrottling(t, 64, FastSync) }
-func testThrottling(t *testing.T, protocol int) {
+func testThrottling(t *testing.T, protocol int, mode SyncMode) {
// Create a long block chain to download and the tester
targetBlocks := 8 * blockCacheLimit
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
- tester.newPeer("peer", protocol, hashes, blocks)
+ tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Wrap the importer to allow stepping
blocked, proceed := uint32(0), make(chan struct{})
- tester.downloader.chainInsertHook = func(blocks []*Block) {
- atomic.StoreUint32(&blocked, uint32(len(blocks)))
+ tester.downloader.chainInsertHook = func(results []*fetchResult) {
+ atomic.StoreUint32(&blocked, uint32(len(results)))
<-proceed
}
// Start a synchronisation concurrently
errc := make(chan error)
go func() {
- errc <- tester.sync("peer", nil)
+ errc <- tester.sync("peer", nil, mode)
}()
// Iteratively take some blocks, always checking the retrieval count
for {
@@ -464,22 +753,37 @@ func testThrottling(t *testing.T, protocol int) {
break
}
// Wait a bit for sync to throttle itself
- var cached int
+ var cached, frozen int
for start := time.Now(); time.Since(start) < time.Second; {
time.Sleep(25 * time.Millisecond)
+ tester.lock.RLock()
tester.downloader.queue.lock.RLock()
- cached = len(tester.downloader.queue.blockPool)
+ cached = len(tester.downloader.queue.blockDonePool)
+ if mode == FastSync {
+ if receipts := len(tester.downloader.queue.receiptDonePool); receipts < cached {
+ if tester.downloader.queue.resultCache[receipts].Header.Number.Uint64() < tester.downloader.queue.fastSyncPivot {
+ cached = receipts
+ }
+ }
+ }
+ frozen = int(atomic.LoadUint32(&blocked))
+ retrieved = len(tester.ownBlocks)
tester.downloader.queue.lock.RUnlock()
+ tester.lock.RUnlock()
- if cached == blockCacheLimit || len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) == targetBlocks+1 {
+ if cached == blockCacheLimit || retrieved+cached+frozen == targetBlocks+1 {
break
}
}
// Make sure we filled up the cache, then exhaust it
time.Sleep(25 * time.Millisecond) // give it a chance to screw up
- if cached != blockCacheLimit && len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) != targetBlocks+1 {
- t.Fatalf("block count mismatch: have %v, want %v (owned %v, target %v)", cached, blockCacheLimit, len(tester.ownBlocks), targetBlocks+1)
+
+ tester.lock.RLock()
+ retrieved = len(tester.ownBlocks)
+ tester.lock.RUnlock()
+ if cached != blockCacheLimit && retrieved+cached+frozen != targetBlocks+1 {
+ t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheLimit, retrieved, frozen, targetBlocks+1)
}
// Permit the blocked blocks to import
if atomic.LoadUint32(&blocked) > 0 {
@@ -488,9 +792,7 @@ func testThrottling(t *testing.T, protocol int) {
}
}
// Check that we haven't pulled more blocks than available
- if len(tester.ownBlocks) > targetBlocks+1 {
- t.Fatalf("target block count mismatch: have %v, want %v", len(tester.ownBlocks), targetBlocks+1)
- }
+ assertOwnChain(t, tester, targetBlocks+1)
if err := <-errc; err != nil {
t.Fatalf("block synchronization failed: %v", err)
}
@@ -499,34 +801,34 @@ func testThrottling(t *testing.T, protocol int) {
// Tests that simple synchronization against a forked chain works correctly. In
// this test common ancestor lookup should *not* be short circuited, and a full
// binary search should be executed.
-func TestForkedSynchronisation61(t *testing.T) { testForkedSynchronisation(t, 61) }
-func TestForkedSynchronisation62(t *testing.T) { testForkedSynchronisation(t, 62) }
-func TestForkedSynchronisation63(t *testing.T) { testForkedSynchronisation(t, 63) }
-func TestForkedSynchronisation64(t *testing.T) { testForkedSynchronisation(t, 64) }
+func TestForkedSynchronisation61(t *testing.T) { testForkedSynchronisation(t, 61, FullSync) }
+func TestForkedSynchronisation62(t *testing.T) { testForkedSynchronisation(t, 62, FullSync) }
+func TestForkedSynchronisation63Full(t *testing.T) { testForkedSynchronisation(t, 63, FullSync) }
+func TestForkedSynchronisation63Fast(t *testing.T) { testForkedSynchronisation(t, 63, FastSync) }
+func TestForkedSynchronisation64Full(t *testing.T) { testForkedSynchronisation(t, 64, FullSync) }
+func TestForkedSynchronisation64Fast(t *testing.T) { testForkedSynchronisation(t, 64, FastSync) }
+func TestForkedSynchronisation64Light(t *testing.T) { testForkedSynchronisation(t, 64, LightSync) }
-func testForkedSynchronisation(t *testing.T, protocol int) {
+func testForkedSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Create a long enough forked chain
common, fork := MaxHashFetch, 2*MaxHashFetch
- hashesA, hashesB, blocksA, blocksB := makeChainFork(common+fork, fork, genesis)
+ hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := makeChainFork(common+fork, fork, genesis, nil)
tester := newTester()
- tester.newPeer("fork A", protocol, hashesA, blocksA)
- tester.newPeer("fork B", protocol, hashesB, blocksB)
+ tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
+ tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
// Synchronise with the peer and make sure all blocks were retrieved
- if err := tester.sync("fork A", nil); err != nil {
+ if err := tester.sync("fork A", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != common+fork+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, common+fork+1)
- }
+ assertOwnChain(t, tester, common+fork+1)
+
// Synchronise with the second peer and make sure that fork is pulled too
- if err := tester.sync("fork B", nil); err != nil {
+ if err := tester.sync("fork B", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != common+2*fork+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, common+2*fork+1)
- }
+ assertOwnForkedChain(t, tester, common+1, []int{common + fork + 1, common + fork + 1})
}
// Tests that an inactive downloader will not accept incoming hashes and blocks.
@@ -534,15 +836,16 @@ func TestInactiveDownloader61(t *testing.T) {
tester := newTester()
// Check that neither hashes nor blocks are accepted
- if err := tester.downloader.DeliverHashes61("bad peer", []common.Hash{}); err != errNoSyncActive {
+ if err := tester.downloader.DeliverHashes("bad peer", []common.Hash{}); err != errNoSyncActive {
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
}
- if err := tester.downloader.DeliverBlocks61("bad peer", []*types.Block{}); err != errNoSyncActive {
+ if err := tester.downloader.DeliverBlocks("bad peer", []*types.Block{}); err != errNoSyncActive {
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
}
}
-// Tests that an inactive downloader will not accept incoming block headers and bodies.
+// Tests that an inactive downloader will not accept incoming block headers and
+// bodies.
func TestInactiveDownloader62(t *testing.T) {
tester := newTester()
@@ -555,13 +858,33 @@ func TestInactiveDownloader62(t *testing.T) {
}
}
-// Tests that a canceled download wipes all previously accumulated state.
-func TestCancel61(t *testing.T) { testCancel(t, 61) }
-func TestCancel62(t *testing.T) { testCancel(t, 62) }
-func TestCancel63(t *testing.T) { testCancel(t, 63) }
-func TestCancel64(t *testing.T) { testCancel(t, 64) }
+// Tests that an inactive downloader will not accept incoming block headers,
+// bodies and receipts.
+func TestInactiveDownloader63(t *testing.T) {
+ tester := newTester()
-func testCancel(t *testing.T, protocol int) {
+ // Check that neither block headers nor bodies are accepted
+ if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive {
+ t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
+ }
+ if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive {
+ t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
+ }
+ if err := tester.downloader.DeliverReceipts("bad peer", [][]*types.Receipt{}); err != errNoSyncActive {
+ t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
+ }
+}
+
+// Tests that a canceled download wipes all previously accumulated state.
+func TestCancel61(t *testing.T) { testCancel(t, 61, FullSync) }
+func TestCancel62(t *testing.T) { testCancel(t, 62, FullSync) }
+func TestCancel63Full(t *testing.T) { testCancel(t, 63, FullSync) }
+func TestCancel63Fast(t *testing.T) { testCancel(t, 63, FastSync) }
+func TestCancel64Full(t *testing.T) { testCancel(t, 64, FullSync) }
+func TestCancel64Fast(t *testing.T) { testCancel(t, 64, FastSync) }
+func TestCancel64Light(t *testing.T) { testCancel(t, 64, LightSync) }
+
+func testCancel(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download and the tester
targetBlocks := blockCacheLimit - 15
if targetBlocks >= MaxHashFetch {
@@ -570,88 +893,80 @@ func testCancel(t *testing.T, protocol int) {
if targetBlocks >= MaxHeaderFetch {
targetBlocks = MaxHeaderFetch - 15
}
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
- tester.newPeer("peer", protocol, hashes, blocks)
+ tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Make sure canceling works with a pristine downloader
tester.downloader.cancel()
- downloading, importing := tester.downloader.queue.Size()
- if downloading > 0 || importing > 0 {
- t.Errorf("download or import count mismatch: %d downloading, %d importing, want 0", downloading, importing)
+ if !tester.downloader.queue.Idle() {
+ t.Errorf("download queue not idle")
}
// Synchronise with the peer, but cancel afterwards
- if err := tester.sync("peer", nil); err != nil {
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
tester.downloader.cancel()
- downloading, importing = tester.downloader.queue.Size()
- if downloading > 0 || importing > 0 {
- t.Errorf("download or import count mismatch: %d downloading, %d importing, want 0", downloading, importing)
+ if !tester.downloader.queue.Idle() {
+ t.Errorf("download queue not idle")
}
}
// Tests that synchronisation from multiple peers works as intended (multi thread sanity test).
-func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, 61) }
-func TestMultiSynchronisation62(t *testing.T) { testMultiSynchronisation(t, 62) }
-func TestMultiSynchronisation63(t *testing.T) { testMultiSynchronisation(t, 63) }
-func TestMultiSynchronisation64(t *testing.T) { testMultiSynchronisation(t, 64) }
+func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, 61, FullSync) }
+func TestMultiSynchronisation62(t *testing.T) { testMultiSynchronisation(t, 62, FullSync) }
+func TestMultiSynchronisation63Full(t *testing.T) { testMultiSynchronisation(t, 63, FullSync) }
+func TestMultiSynchronisation63Fast(t *testing.T) { testMultiSynchronisation(t, 63, FastSync) }
+func TestMultiSynchronisation64Full(t *testing.T) { testMultiSynchronisation(t, 64, FullSync) }
+func TestMultiSynchronisation64Fast(t *testing.T) { testMultiSynchronisation(t, 64, FastSync) }
+func TestMultiSynchronisation64Light(t *testing.T) { testMultiSynchronisation(t, 64, LightSync) }
-func testMultiSynchronisation(t *testing.T, protocol int) {
+func testMultiSynchronisation(t *testing.T, protocol int, mode SyncMode) {
// Create various peers with various parts of the chain
targetPeers := 8
targetBlocks := targetPeers*blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
for i := 0; i < targetPeers; i++ {
id := fmt.Sprintf("peer #%d", i)
- tester.newPeer(id, protocol, hashes[i*blockCacheLimit:], blocks)
+ tester.newPeer(id, protocol, hashes[i*blockCacheLimit:], headers, blocks, receipts)
}
- // Synchronise with the middle peer and make sure half of the blocks were retrieved
- id := fmt.Sprintf("peer #%d", targetPeers/2)
- if err := tester.sync(id, nil); err != nil {
+ if err := tester.sync("peer #0", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != len(tester.peerHashes[id]) {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(tester.peerHashes[id]))
- }
- // Synchronise with the best peer and make sure everything is retrieved
- if err := tester.sync("peer #0", nil); err != nil {
- t.Fatalf("failed to synchronise blocks: %v", err)
- }
- if imported := len(tester.ownBlocks); imported != targetBlocks+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, targetBlocks+1)
- }
+ assertOwnChain(t, tester, targetBlocks+1)
}
// Tests that synchronisations behave well in multi-version protocol environments
// and not wreak havok on other nodes in the network.
-func TestMultiProtocolSynchronisation61(t *testing.T) { testMultiProtocolSynchronisation(t, 61) }
-func TestMultiProtocolSynchronisation62(t *testing.T) { testMultiProtocolSynchronisation(t, 62) }
-func TestMultiProtocolSynchronisation63(t *testing.T) { testMultiProtocolSynchronisation(t, 63) }
-func TestMultiProtocolSynchronisation64(t *testing.T) { testMultiProtocolSynchronisation(t, 64) }
+func TestMultiProtoSynchronisation61(t *testing.T) { testMultiProtoSync(t, 61, FullSync) }
+func TestMultiProtoSynchronisation62(t *testing.T) { testMultiProtoSync(t, 62, FullSync) }
+func TestMultiProtoSynchronisation63Full(t *testing.T) { testMultiProtoSync(t, 63, FullSync) }
+func TestMultiProtoSynchronisation63Fast(t *testing.T) { testMultiProtoSync(t, 63, FastSync) }
+func TestMultiProtoSynchronisation64Full(t *testing.T) { testMultiProtoSync(t, 64, FullSync) }
+func TestMultiProtoSynchronisation64Fast(t *testing.T) { testMultiProtoSync(t, 64, FastSync) }
+func TestMultiProtoSynchronisation64Light(t *testing.T) { testMultiProtoSync(t, 64, LightSync) }
-func testMultiProtocolSynchronisation(t *testing.T, protocol int) {
+func testMultiProtoSync(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
// Create peers of every type
tester := newTester()
- tester.newPeer("peer 61", 61, hashes, blocks)
- tester.newPeer("peer 62", 62, hashes, blocks)
- tester.newPeer("peer 63", 63, hashes, blocks)
- tester.newPeer("peer 64", 64, hashes, blocks)
+ tester.newPeer("peer 61", 61, hashes, nil, blocks, nil)
+ tester.newPeer("peer 62", 62, hashes, headers, blocks, nil)
+ tester.newPeer("peer 63", 63, hashes, headers, blocks, receipts)
+ tester.newPeer("peer 64", 64, hashes, headers, blocks, receipts)
- // Synchronise with the requestd peer and make sure all blocks were retrieved
- if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil); err != nil {
+ // Synchronise with the requested peer and make sure all blocks were retrieved
+ if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != targetBlocks+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, targetBlocks+1)
- }
+ assertOwnChain(t, tester, targetBlocks+1)
+
// Check that no peers have been dropped off
for _, version := range []int{61, 62, 63, 64} {
peer := fmt.Sprintf("peer %d", version)
@@ -661,151 +976,223 @@ func testMultiProtocolSynchronisation(t *testing.T, protocol int) {
}
}
-// Tests that if a block is empty (i.e. header only), no body request should be
+// Tests that if a block is empty (e.g. header only), no body request should be
// made, and instead the header should be assembled into a whole block in itself.
-func TestEmptyBlockShortCircuit62(t *testing.T) { testEmptyBlockShortCircuit(t, 62) }
-func TestEmptyBlockShortCircuit63(t *testing.T) { testEmptyBlockShortCircuit(t, 63) }
-func TestEmptyBlockShortCircuit64(t *testing.T) { testEmptyBlockShortCircuit(t, 64) }
+func TestEmptyShortCircuit62(t *testing.T) { testEmptyShortCircuit(t, 62, FullSync) }
+func TestEmptyShortCircuit63Full(t *testing.T) { testEmptyShortCircuit(t, 63, FullSync) }
+func TestEmptyShortCircuit63Fast(t *testing.T) { testEmptyShortCircuit(t, 63, FastSync) }
+func TestEmptyShortCircuit64Full(t *testing.T) { testEmptyShortCircuit(t, 64, FullSync) }
+func TestEmptyShortCircuit64Fast(t *testing.T) { testEmptyShortCircuit(t, 64, FastSync) }
+func TestEmptyShortCircuit64Light(t *testing.T) { testEmptyShortCircuit(t, 64, LightSync) }
-func testEmptyBlockShortCircuit(t *testing.T, protocol int) {
- // Create a small enough block chain to download
- targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+func testEmptyShortCircuit(t *testing.T, protocol int, mode SyncMode) {
+ // Create a block chain to download
+ targetBlocks := 2*blockCacheLimit - 15
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
- tester.newPeer("peer", protocol, hashes, blocks)
+ tester.newPeer("peer", protocol, hashes, headers, blocks, receipts)
// Instrument the downloader to signal body requests
- requested := int32(0)
+ bodiesHave, receiptsHave := int32(0), int32(0)
tester.downloader.bodyFetchHook = func(headers []*types.Header) {
- atomic.AddInt32(&requested, int32(len(headers)))
+ atomic.AddInt32(&bodiesHave, int32(len(headers)))
+ }
+ tester.downloader.receiptFetchHook = func(headers []*types.Header) {
+ atomic.AddInt32(&receiptsHave, int32(len(headers)))
}
// Synchronise with the peer and make sure all blocks were retrieved
- if err := tester.sync("peer", nil); err != nil {
+ if err := tester.sync("peer", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != targetBlocks+1 {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, targetBlocks+1)
- }
+ assertOwnChain(t, tester, targetBlocks+1)
+
// Validate the number of block bodies that should have been requested
- needed := 0
+ bodiesNeeded, receiptsNeeded := 0, 0
for _, block := range blocks {
- if block != genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) {
- needed++
+ if mode != LightSync && block != genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) {
+ bodiesNeeded++
}
}
- if int(requested) != needed {
- t.Fatalf("block body retrieval count mismatch: have %v, want %v", requested, needed)
+ for hash, receipt := range receipts {
+ if mode == FastSync && len(receipt) > 0 && headers[hash].Number.Uint64() <= tester.downloader.queue.fastSyncPivot {
+ receiptsNeeded++
+ }
+ }
+ if int(bodiesHave) != bodiesNeeded {
+ t.Errorf("body retrieval count mismatch: have %v, want %v", bodiesHave, bodiesNeeded)
+ }
+ if int(receiptsHave) != receiptsNeeded {
+ t.Errorf("receipt retrieval count mismatch: have %v, want %v", receiptsHave, receiptsNeeded)
}
}
// Tests that headers are enqueued continuously, preventing malicious nodes from
// stalling the downloader by feeding gapped header chains.
-func TestMissingHeaderAttack62(t *testing.T) { testMissingHeaderAttack(t, 62) }
-func TestMissingHeaderAttack63(t *testing.T) { testMissingHeaderAttack(t, 63) }
-func TestMissingHeaderAttack64(t *testing.T) { testMissingHeaderAttack(t, 64) }
+func TestMissingHeaderAttack62(t *testing.T) { testMissingHeaderAttack(t, 62, FullSync) }
+func TestMissingHeaderAttack63Full(t *testing.T) { testMissingHeaderAttack(t, 63, FullSync) }
+func TestMissingHeaderAttack63Fast(t *testing.T) { testMissingHeaderAttack(t, 63, FastSync) }
+func TestMissingHeaderAttack64Full(t *testing.T) { testMissingHeaderAttack(t, 64, FullSync) }
+func TestMissingHeaderAttack64Fast(t *testing.T) { testMissingHeaderAttack(t, 64, FastSync) }
+func TestMissingHeaderAttack64Light(t *testing.T) { testMissingHeaderAttack(t, 64, LightSync) }
-func testMissingHeaderAttack(t *testing.T, protocol int) {
+func testMissingHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
// Attempt a full sync with an attacker feeding gapped headers
- tester.newPeer("attack", protocol, hashes, blocks)
+ tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
missing := targetBlocks / 2
- delete(tester.peerBlocks["attack"], hashes[missing])
+ delete(tester.peerHeaders["attack"], hashes[missing])
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
- tester.newPeer("valid", protocol, hashes, blocks)
- if err := tester.sync("valid", nil); err != nil {
+ tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != len(hashes) {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
- }
+ assertOwnChain(t, tester, targetBlocks+1)
}
// Tests that if requested headers are shifted (i.e. first is missing), the queue
// detects the invalid numbering.
-func TestShiftedHeaderAttack62(t *testing.T) { testShiftedHeaderAttack(t, 62) }
-func TestShiftedHeaderAttack63(t *testing.T) { testShiftedHeaderAttack(t, 63) }
-func TestShiftedHeaderAttack64(t *testing.T) { testShiftedHeaderAttack(t, 64) }
+func TestShiftedHeaderAttack62(t *testing.T) { testShiftedHeaderAttack(t, 62, FullSync) }
+func TestShiftedHeaderAttack63Full(t *testing.T) { testShiftedHeaderAttack(t, 63, FullSync) }
+func TestShiftedHeaderAttack63Fast(t *testing.T) { testShiftedHeaderAttack(t, 63, FastSync) }
+func TestShiftedHeaderAttack64Full(t *testing.T) { testShiftedHeaderAttack(t, 64, FullSync) }
+func TestShiftedHeaderAttack64Fast(t *testing.T) { testShiftedHeaderAttack(t, 64, FastSync) }
+func TestShiftedHeaderAttack64Light(t *testing.T) { testShiftedHeaderAttack(t, 64, LightSync) }
-func testShiftedHeaderAttack(t *testing.T, protocol int) {
+func testShiftedHeaderAttack(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
// Attempt a full sync with an attacker feeding shifted headers
- tester.newPeer("attack", protocol, hashes, blocks)
+ tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
+ delete(tester.peerHeaders["attack"], hashes[len(hashes)-2])
delete(tester.peerBlocks["attack"], hashes[len(hashes)-2])
+ delete(tester.peerReceipts["attack"], hashes[len(hashes)-2])
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
// Synchronise with the valid peer and make sure sync succeeds
- tester.newPeer("valid", protocol, hashes, blocks)
- if err := tester.sync("valid", nil); err != nil {
+ tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != len(hashes) {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
- }
+ assertOwnChain(t, tester, targetBlocks+1)
}
-// Tests that if a peer sends an invalid body for a requested block, it gets
-// dropped immediately by the downloader.
-func TestInvalidBlockBodyAttack62(t *testing.T) { testInvalidBlockBodyAttack(t, 62) }
-func TestInvalidBlockBodyAttack63(t *testing.T) { testInvalidBlockBodyAttack(t, 63) }
-func TestInvalidBlockBodyAttack64(t *testing.T) { testInvalidBlockBodyAttack(t, 64) }
+// Tests that upon detecting an invalid header, the recent ones are rolled back
+func TestInvalidHeaderRollback63Fast(t *testing.T) { testInvalidHeaderRollback(t, 63, FastSync) }
+func TestInvalidHeaderRollback64Fast(t *testing.T) { testInvalidHeaderRollback(t, 64, FastSync) }
+func TestInvalidHeaderRollback64Light(t *testing.T) { testInvalidHeaderRollback(t, 64, LightSync) }
-func testInvalidBlockBodyAttack(t *testing.T, protocol int) {
- // Create two peers, one feeding invalid block bodies
- targetBlocks := 4*blockCacheLimit - 15
- hashes, validBlocks := makeChain(targetBlocks, 0, genesis)
-
- invalidBlocks := make(map[common.Hash]*types.Block)
- for hash, block := range validBlocks {
- invalidBlocks[hash] = types.NewBlockWithHeader(block.Header())
- }
+func testInvalidHeaderRollback(t *testing.T, protocol int, mode SyncMode) {
+ // Create a small enough block chain to download
+ targetBlocks := 3*fsHeaderSafetyNet + fsMinFullBlocks
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
tester := newTester()
- tester.newPeer("valid", protocol, hashes, validBlocks)
- tester.newPeer("attack", protocol, hashes, invalidBlocks)
- // Synchronise with the valid peer (will pull contents from the attacker too)
- if err := tester.sync("valid", nil); err != nil {
+ // Attempt to sync with an attacker that feeds junk during the fast sync phase.
+ // This should result in the last fsHeaderSafetyNet headers being rolled back.
+ tester.newPeer("fast-attack", protocol, hashes, headers, blocks, receipts)
+ missing := fsHeaderSafetyNet + MaxHeaderFetch + 1
+ delete(tester.peerHeaders["fast-attack"], hashes[len(hashes)-missing])
+
+ if err := tester.sync("fast-attack", nil, mode); err == nil {
+ t.Fatalf("succeeded fast attacker synchronisation")
+ }
+ if head := tester.headHeader().Number.Int64(); int(head) > MaxHeaderFetch {
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch)
+ }
+ // Attempt to sync with an attacker that feeds junk during the block import phase.
+ // This should result in both the last fsHeaderSafetyNet number of headers being
+ // rolled back, and also the pivot point being reverted to a non-block status.
+ tester.newPeer("block-attack", protocol, hashes, headers, blocks, receipts)
+ missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
+ delete(tester.peerHeaders["block-attack"], hashes[len(hashes)-missing])
+
+ if err := tester.sync("block-attack", nil, mode); err == nil {
+ t.Fatalf("succeeded block attacker synchronisation")
+ }
+ if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
+ }
+ if mode == FastSync {
+ if head := tester.headBlock().NumberU64(); head != 0 {
+ t.Errorf("fast sync pivot block #%d not rolled back", head)
+ }
+ }
+ // Attempt to sync with an attacker that withholds promised blocks after the
+ // fast sync pivot point. This could be a trial to leave the node with a bad
+ // but already imported pivot block.
+ tester.newPeer("withhold-attack", protocol, hashes, headers, blocks, receipts)
+ missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1
+
+ tester.downloader.noFast = false
+ tester.downloader.syncInitHook = func(uint64, uint64) {
+ for i := missing; i <= len(hashes); i++ {
+ delete(tester.peerHeaders["withhold-attack"], hashes[len(hashes)-i])
+ }
+ tester.downloader.syncInitHook = nil
+ }
+
+ if err := tester.sync("withhold-attack", nil, mode); err == nil {
+ t.Fatalf("succeeded withholding attacker synchronisation")
+ }
+ if head := tester.headHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch {
+ t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch)
+ }
+ if mode == FastSync {
+ if head := tester.headBlock().NumberU64(); head != 0 {
+ t.Errorf("fast sync pivot block #%d not rolled back", head)
+ }
+ }
+ // Synchronise with the valid peer and make sure sync succeeds. Since the last
+ // rollback should also disable fast syncing for this process, verify that we
+ // did a fresh full sync. Note, we can't assert anything about the receipts
+ // since we won't purge the database of them, hence we can't use asserOwnChain.
+ tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
- if imported := len(tester.ownBlocks); imported != len(hashes) {
- t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
+ if hs := len(tester.ownHeaders); hs != len(headers) {
+ t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, len(headers))
}
- // Make sure the attacker was detected and dropped in the mean time
- if _, ok := tester.peerHashes["attack"]; ok {
- t.Fatalf("block body attacker not detected/dropped")
+ if mode != LightSync {
+ if bs := len(tester.ownBlocks); bs != len(blocks) {
+ t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, len(blocks))
+ }
}
}
// Tests that a peer advertising an high TD doesn't get to stall the downloader
// afterwards by not sending any useful hashes.
-func TestHighTDStarvationAttack61(t *testing.T) { testHighTDStarvationAttack(t, 61) }
-func TestHighTDStarvationAttack62(t *testing.T) { testHighTDStarvationAttack(t, 62) }
-func TestHighTDStarvationAttack63(t *testing.T) { testHighTDStarvationAttack(t, 63) }
-func TestHighTDStarvationAttack64(t *testing.T) { testHighTDStarvationAttack(t, 64) }
+func TestHighTDStarvationAttack61(t *testing.T) { testHighTDStarvationAttack(t, 61, FullSync) }
+func TestHighTDStarvationAttack62(t *testing.T) { testHighTDStarvationAttack(t, 62, FullSync) }
+func TestHighTDStarvationAttack63Full(t *testing.T) { testHighTDStarvationAttack(t, 63, FullSync) }
+func TestHighTDStarvationAttack63Fast(t *testing.T) { testHighTDStarvationAttack(t, 63, FastSync) }
+func TestHighTDStarvationAttack64Full(t *testing.T) { testHighTDStarvationAttack(t, 64, FullSync) }
+func TestHighTDStarvationAttack64Fast(t *testing.T) { testHighTDStarvationAttack(t, 64, FastSync) }
+func TestHighTDStarvationAttack64Light(t *testing.T) { testHighTDStarvationAttack(t, 64, LightSync) }
-func testHighTDStarvationAttack(t *testing.T, protocol int) {
+func testHighTDStarvationAttack(t *testing.T, protocol int, mode SyncMode) {
tester := newTester()
- hashes, blocks := makeChain(0, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(0, 0, genesis, nil)
- tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, blocks)
- if err := tester.sync("attack", big.NewInt(1000000)); err != errStallingPeer {
+ tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, headers, blocks, receipts)
+ if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer {
t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer)
}
}
@@ -834,7 +1221,9 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
{errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end
{errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser
{errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop
+ {errInvalidBlock, false}, // A bad peer was detected, but not the sync origin
{errInvalidBody, false}, // A bad peer was detected, but not the sync origin
+ {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin
{errCancelHashFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
{errCancelBlockFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
{errCancelHeaderFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
@@ -845,7 +1234,7 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
for i, tt := range tests {
// Register a new peer and ensure it's presence
id := fmt.Sprintf("test %d", i)
- if err := tester.newPeer(id, protocol, []common.Hash{genesis.Hash()}, nil); err != nil {
+ if err := tester.newPeer(id, protocol, []common.Hash{genesis.Hash()}, nil, nil, nil); err != nil {
t.Fatalf("test %d: failed to register new peer: %v", i, err)
}
if _, ok := tester.peerHashes[id]; !ok {
@@ -854,70 +1243,29 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
// Simulate a synchronisation and check the required result
tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result }
- tester.downloader.Synchronise(id, genesis.Hash(), big.NewInt(1000))
+ tester.downloader.Synchronise(id, genesis.Hash(), big.NewInt(1000), FullSync)
if _, ok := tester.peerHashes[id]; !ok != tt.drop {
t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop)
}
}
}
-// Tests that feeding bad blocks will result in a peer drop.
-func TestBlockBodyAttackerDropping61(t *testing.T) { testBlockBodyAttackerDropping(t, 61) }
-func TestBlockBodyAttackerDropping62(t *testing.T) { testBlockBodyAttackerDropping(t, 62) }
-func TestBlockBodyAttackerDropping63(t *testing.T) { testBlockBodyAttackerDropping(t, 63) }
-func TestBlockBodyAttackerDropping64(t *testing.T) { testBlockBodyAttackerDropping(t, 64) }
+// Tests that synchronisation progress (origin block number, current block number
+// and highest block number) is tracked and updated correctly.
+func TestSyncProgress61(t *testing.T) { testSyncProgress(t, 61, FullSync) }
+func TestSyncProgress62(t *testing.T) { testSyncProgress(t, 62, FullSync) }
+func TestSyncProgress63Full(t *testing.T) { testSyncProgress(t, 63, FullSync) }
+func TestSyncProgress63Fast(t *testing.T) { testSyncProgress(t, 63, FastSync) }
+func TestSyncProgress64Full(t *testing.T) { testSyncProgress(t, 64, FullSync) }
+func TestSyncProgress64Fast(t *testing.T) { testSyncProgress(t, 64, FastSync) }
+func TestSyncProgress64Light(t *testing.T) { testSyncProgress(t, 64, LightSync) }
-func testBlockBodyAttackerDropping(t *testing.T, protocol int) {
- // Define the disconnection requirement for individual block import errors
- tests := []struct {
- failure bool
- drop bool
- }{
- {true, true},
- {false, false},
- }
-
- // Run the tests and check disconnection status
- tester := newTester()
- for i, tt := range tests {
- // Register a new peer and ensure it's presence
- id := fmt.Sprintf("test %d", i)
- if err := tester.newPeer(id, protocol, []common.Hash{common.Hash{}}, nil); err != nil {
- t.Fatalf("test %d: failed to register new peer: %v", i, err)
- }
- if _, ok := tester.peerHashes[id]; !ok {
- t.Fatalf("test %d: registered peer not found", i)
- }
- // Assemble a good or bad block, depending of the test
- raw := core.GenerateChain(genesis, testdb, 1, nil)[0]
- if tt.failure {
- parent := types.NewBlock(&types.Header{}, nil, nil, nil)
- raw = core.GenerateChain(parent, testdb, 1, nil)[0]
- }
- block := &Block{OriginPeer: id, RawBlock: raw}
-
- // Simulate block processing and check the result
- tester.downloader.queue.blockCache[0] = block
- tester.downloader.process()
- if _, ok := tester.peerHashes[id]; !ok != tt.drop {
- t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.failure, !ok, tt.drop)
- }
- }
-}
-
-// Tests that synchronisation boundaries (origin block number and highest block
-// number) is tracked and updated correctly.
-func TestSyncBoundaries61(t *testing.T) { testSyncBoundaries(t, 61) }
-func TestSyncBoundaries62(t *testing.T) { testSyncBoundaries(t, 62) }
-func TestSyncBoundaries63(t *testing.T) { testSyncBoundaries(t, 63) }
-func TestSyncBoundaries64(t *testing.T) { testSyncBoundaries(t, 64) }
-
-func testSyncBoundaries(t *testing.T, protocol int) {
+func testSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
@@ -926,60 +1274,68 @@ func testSyncBoundaries(t *testing.T, protocol int) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
- // Synchronise half the blocks and check initial boundaries
- tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], blocks)
+ // Synchronise half the blocks and check initial progress
+ tester.newPeer("peer-half", protocol, hashes[targetBlocks/2:], headers, blocks, receipts)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("peer-half", nil); err != nil {
+ if err := tester.sync("peer-half", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks/2+1) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks/2+1)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks/2+1) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks/2+1)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise all the blocks and check continuation boundaries
- tester.newPeer("peer-full", protocol, hashes, blocks)
+ // Synchronise all the blocks and check continuation progress
+ tester.newPeer("peer-full", protocol, hashes, headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("peer-full", nil); err != nil {
+ if err := tester.sync("peer-full", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
- t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, targetBlocks/2+1, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks/2+1) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks/2+1, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(targetBlocks/2+1) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, targetBlocks/2+1, targetBlocks, targetBlocks)
+ }
}
-// Tests that synchronisation boundaries (origin block number and highest block
+// Tests that synchronisation progress (origin block number and highest block
// number) is tracked and updated correctly in case of a fork (or manual head
// revertal).
-func TestForkedSyncBoundaries61(t *testing.T) { testForkedSyncBoundaries(t, 61) }
-func TestForkedSyncBoundaries62(t *testing.T) { testForkedSyncBoundaries(t, 62) }
-func TestForkedSyncBoundaries63(t *testing.T) { testForkedSyncBoundaries(t, 63) }
-func TestForkedSyncBoundaries64(t *testing.T) { testForkedSyncBoundaries(t, 64) }
+func TestForkedSyncProgress61(t *testing.T) { testForkedSyncProgress(t, 61, FullSync) }
+func TestForkedSyncProgress62(t *testing.T) { testForkedSyncProgress(t, 62, FullSync) }
+func TestForkedSyncProgress63Full(t *testing.T) { testForkedSyncProgress(t, 63, FullSync) }
+func TestForkedSyncProgress63Fast(t *testing.T) { testForkedSyncProgress(t, 63, FastSync) }
+func TestForkedSyncProgress64Full(t *testing.T) { testForkedSyncProgress(t, 64, FullSync) }
+func TestForkedSyncProgress64Fast(t *testing.T) { testForkedSyncProgress(t, 64, FastSync) }
+func TestForkedSyncProgress64Light(t *testing.T) { testForkedSyncProgress(t, 64, LightSync) }
-func testForkedSyncBoundaries(t *testing.T, protocol int) {
+func testForkedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a forked chain to simulate origin revertal
common, fork := MaxHashFetch, 2*MaxHashFetch
- hashesA, hashesB, blocksA, blocksB := makeChainFork(common+fork, fork, genesis)
+ hashesA, hashesB, headersA, headersB, blocksA, blocksB, receiptsA, receiptsB := makeChainFork(common+fork, fork, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
@@ -988,63 +1344,71 @@ func testForkedSyncBoundaries(t *testing.T, protocol int) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
- // Synchronise with one of the forks and check boundaries
- tester.newPeer("fork A", protocol, hashesA, blocksA)
+ // Synchronise with one of the forks and check progress
+ tester.newPeer("fork A", protocol, hashesA, headersA, blocksA, receiptsA)
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("fork A", nil); err != nil {
+ if err := tester.sync("fork A", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(len(hashesA)-1) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, len(hashesA)-1)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(len(hashesA)-1) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, len(hashesA)-1)
}
progress <- struct{}{}
pending.Wait()
// Simulate a successful sync above the fork
- tester.downloader.syncStatsOrigin = tester.downloader.syncStatsHeight
+ tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight
- // Synchronise with the second fork and check boundary resets
- tester.newPeer("fork B", protocol, hashesB, blocksB)
+ // Synchronise with the second fork and check progress resets
+ tester.newPeer("fork B", protocol, hashesB, headersB, blocksB, receiptsB)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("fork B", nil); err != nil {
+ if err := tester.sync("fork B", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != uint64(common) || latest != uint64(len(hashesB)-1) {
- t.Fatalf("Forking boundary mismatch: have %v/%v, want %v/%v", origin, latest, common, len(hashesB)-1)
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesA)-1) || latest != uint64(len(hashesB)-1) {
+ t.Fatalf("Forking progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesA)-1, len(hashesB)-1)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin != uint64(common) || current != uint64(len(hashesB)-1) || latest != uint64(len(hashesB)-1) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, common, len(hashesB)-1, len(hashesB)-1)
+ }
}
-// Tests that if synchronisation is aborted due to some failure, then the boundary
+// Tests that if synchronisation is aborted due to some failure, then the progress
// origin is not updated in the next sync cycle, as it should be considered the
// continuation of the previous sync and not a new instance.
-func TestFailedSyncBoundaries61(t *testing.T) { testFailedSyncBoundaries(t, 61) }
-func TestFailedSyncBoundaries62(t *testing.T) { testFailedSyncBoundaries(t, 62) }
-func TestFailedSyncBoundaries63(t *testing.T) { testFailedSyncBoundaries(t, 63) }
-func TestFailedSyncBoundaries64(t *testing.T) { testFailedSyncBoundaries(t, 64) }
+func TestFailedSyncProgress61(t *testing.T) { testFailedSyncProgress(t, 61, FullSync) }
+func TestFailedSyncProgress62(t *testing.T) { testFailedSyncProgress(t, 62, FullSync) }
+func TestFailedSyncProgress63Full(t *testing.T) { testFailedSyncProgress(t, 63, FullSync) }
+func TestFailedSyncProgress63Fast(t *testing.T) { testFailedSyncProgress(t, 63, FastSync) }
+func TestFailedSyncProgress64Full(t *testing.T) { testFailedSyncProgress(t, 64, FullSync) }
+func TestFailedSyncProgress64Fast(t *testing.T) { testFailedSyncProgress(t, 64, FastSync) }
+func TestFailedSyncProgress64Light(t *testing.T) { testFailedSyncProgress(t, 64, LightSync) }
-func testFailedSyncBoundaries(t *testing.T, protocol int) {
+func testFailedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small enough block chain to download
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
@@ -1053,62 +1417,72 @@ func testFailedSyncBoundaries(t *testing.T, protocol int) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
// Attempt a full sync with a faulty peer
- tester.newPeer("faulty", protocol, hashes, blocks)
+ tester.newPeer("faulty", protocol, hashes, headers, blocks, receipts)
missing := targetBlocks / 2
+ delete(tester.peerHeaders["faulty"], hashes[missing])
delete(tester.peerBlocks["faulty"], hashes[missing])
+ delete(tester.peerReceipts["faulty"], hashes[missing])
pending := new(sync.WaitGroup)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("faulty", nil); err == nil {
+ if err := tester.sync("faulty", nil, mode); err == nil {
t.Fatalf("succeeded faulty synchronisation")
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise with a good peer and check that the boundary origin remind the same after a failure
- tester.newPeer("valid", protocol, hashes, blocks)
+ // Synchronise with a good peer and check that the progress origin remind the same after a failure
+ tester.newPeer("valid", protocol, hashes, headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Completing boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks/2) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks/2, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks/2) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks/2, targetBlocks, targetBlocks)
+ }
}
// Tests that if an attacker fakes a chain height, after the attack is detected,
-// the boundary height is successfully reduced at the next sync invocation.
-func TestFakedSyncBoundaries61(t *testing.T) { testFakedSyncBoundaries(t, 61) }
-func TestFakedSyncBoundaries62(t *testing.T) { testFakedSyncBoundaries(t, 62) }
-func TestFakedSyncBoundaries63(t *testing.T) { testFakedSyncBoundaries(t, 63) }
-func TestFakedSyncBoundaries64(t *testing.T) { testFakedSyncBoundaries(t, 64) }
+// the progress height is successfully reduced at the next sync invocation.
+func TestFakedSyncProgress61(t *testing.T) { testFakedSyncProgress(t, 61, FullSync) }
+func TestFakedSyncProgress62(t *testing.T) { testFakedSyncProgress(t, 62, FullSync) }
+func TestFakedSyncProgress63Full(t *testing.T) { testFakedSyncProgress(t, 63, FullSync) }
+func TestFakedSyncProgress63Fast(t *testing.T) { testFakedSyncProgress(t, 63, FastSync) }
+func TestFakedSyncProgress64Full(t *testing.T) { testFakedSyncProgress(t, 64, FullSync) }
+func TestFakedSyncProgress64Fast(t *testing.T) { testFakedSyncProgress(t, 64, FastSync) }
+func TestFakedSyncProgress64Light(t *testing.T) { testFakedSyncProgress(t, 64, LightSync) }
-func testFakedSyncBoundaries(t *testing.T, protocol int) {
+func testFakedSyncProgress(t *testing.T, protocol int, mode SyncMode) {
// Create a small block chain
targetBlocks := blockCacheLimit - 15
- hashes, blocks := makeChain(targetBlocks+3, 0, genesis)
+ hashes, headers, blocks, receipts := makeChain(targetBlocks+3, 0, genesis, nil)
- // Set a sync init hook to catch boundary changes
+ // Set a sync init hook to catch progress changes
starting := make(chan struct{})
progress := make(chan struct{})
@@ -1117,14 +1491,16 @@ func testFakedSyncBoundaries(t *testing.T, protocol int) {
starting <- struct{}{}
<-progress
}
- // Retrieve the sync boundaries and ensure they are zero (pristine sync)
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != 0 {
- t.Fatalf("Pristine boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, 0)
+ // Retrieve the sync progress and ensure they are zero (pristine sync)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != 0 {
+ t.Fatalf("Pristine progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, 0)
}
// Create and sync with an attacker that promises a higher chain than available
- tester.newPeer("attack", protocol, hashes, blocks)
+ tester.newPeer("attack", protocol, hashes, headers, blocks, receipts)
for i := 1; i < 3; i++ {
+ delete(tester.peerHeaders["attack"], hashes[i])
delete(tester.peerBlocks["attack"], hashes[i])
+ delete(tester.peerReceipts["attack"], hashes[i])
}
pending := new(sync.WaitGroup)
@@ -1132,31 +1508,36 @@ func testFakedSyncBoundaries(t *testing.T, protocol int) {
go func() {
defer pending.Done()
- if err := tester.sync("attack", nil); err == nil {
+ if err := tester.sync("attack", nil, mode); err == nil {
t.Fatalf("succeeded attacker synchronisation")
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks+3) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks+3)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current != 0 || latest != uint64(targetBlocks+3) {
+ t.Fatalf("Initial progress mismatch: have %v/%v/%v, want %v/%v/%v", origin, current, latest, 0, 0, targetBlocks+3)
}
progress <- struct{}{}
pending.Wait()
- // Synchronise with a good peer and check that the boundary height has been reduced to the true value
- tester.newPeer("valid", protocol, hashes[3:], blocks)
+ // Synchronise with a good peer and check that the progress height has been reduced to the true value
+ tester.newPeer("valid", protocol, hashes[3:], headers, blocks, receipts)
pending.Add(1)
go func() {
defer pending.Done()
- if err := tester.sync("valid", nil); err != nil {
+ if err := tester.sync("valid", nil, mode); err != nil {
t.Fatalf("failed to synchronise blocks: %v", err)
}
}()
<-starting
- if origin, latest := tester.downloader.Boundaries(); origin != 0 || latest != uint64(targetBlocks) {
- t.Fatalf("Initial boundary mismatch: have %v/%v, want %v/%v", origin, latest, 0, targetBlocks)
+ if origin, current, latest := tester.downloader.Progress(); origin != 0 || current > uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Completing progress mismatch: have %v/%v/%v, want %v/0-%v/%v", origin, current, latest, 0, targetBlocks, targetBlocks)
}
progress <- struct{}{}
pending.Wait()
+
+ // Check final progress after successful sync
+ if origin, current, latest := tester.downloader.Progress(); origin > uint64(targetBlocks) || current != uint64(targetBlocks) || latest != uint64(targetBlocks) {
+ t.Fatalf("Final progress mismatch: have %v/%v/%v, want 0-%v/%v/%v", origin, current, latest, targetBlocks, targetBlocks, targetBlocks)
+ }
}
diff --git a/eth/downloader/metrics.go b/eth/downloader/metrics.go
index fd926affd..d6fcfa25c 100644
--- a/eth/downloader/metrics.go
+++ b/eth/downloader/metrics.go
@@ -42,4 +42,14 @@ var (
bodyReqTimer = metrics.NewTimer("eth/downloader/bodies/req")
bodyDropMeter = metrics.NewMeter("eth/downloader/bodies/drop")
bodyTimeoutMeter = metrics.NewMeter("eth/downloader/bodies/timeout")
+
+ receiptInMeter = metrics.NewMeter("eth/downloader/receipts/in")
+ receiptReqTimer = metrics.NewTimer("eth/downloader/receipts/req")
+ receiptDropMeter = metrics.NewMeter("eth/downloader/receipts/drop")
+ receiptTimeoutMeter = metrics.NewMeter("eth/downloader/receipts/timeout")
+
+ stateInMeter = metrics.NewMeter("eth/downloader/states/in")
+ stateReqTimer = metrics.NewTimer("eth/downloader/states/req")
+ stateDropMeter = metrics.NewMeter("eth/downloader/states/drop")
+ stateTimeoutMeter = metrics.NewMeter("eth/downloader/states/timeout")
)
diff --git a/eth/downloader/modes.go b/eth/downloader/modes.go
new file mode 100644
index 000000000..ec339c074
--- /dev/null
+++ b/eth/downloader/modes.go
@@ -0,0 +1,26 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package downloader
+
+// SyncMode represents the synchronisation mode of the downloader.
+type SyncMode int
+
+const (
+ FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks
+ FastSync // Quickly download the headers, full sync only at the chain head
+ LightSync // Download only the headers and terminate afterwards
+)
diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go
index c1d20ac61..1f457cb15 100644
--- a/eth/downloader/peer.go
+++ b/eth/downloader/peer.go
@@ -36,10 +36,12 @@ type relativeHashFetcherFn func(common.Hash) error
type absoluteHashFetcherFn func(uint64, int) error
type blockFetcherFn func([]common.Hash) error
-// Block header and body fethers belonging to eth/62 and above
+// Block header and body fetchers belonging to eth/62 and above
type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error
type absoluteHeaderFetcherFn func(uint64, int, int, bool) error
type blockBodyFetcherFn func([]common.Hash) error
+type receiptFetcherFn func([]common.Hash) error
+type stateFetcherFn func([]common.Hash) error
var (
errAlreadyFetching = errors.New("already fetching blocks from peer")
@@ -52,11 +54,18 @@ type peer struct {
id string // Unique identifier of the peer
head common.Hash // Hash of the peers latest known block
- idle int32 // Current activity state of the peer (idle = 0, active = 1)
- rep int32 // Simple peer reputation
+ blockIdle int32 // Current block activity state of the peer (idle = 0, active = 1)
+ receiptIdle int32 // Current receipt activity state of the peer (idle = 0, active = 1)
+ stateIdle int32 // Current node data activity state of the peer (idle = 0, active = 1)
+ rep int32 // Simple peer reputation
- capacity int32 // Number of blocks allowed to fetch per request
- started time.Time // Time instance when the last fetch was started
+ blockCapacity int32 // Number of blocks (bodies) allowed to fetch per request
+ receiptCapacity int32 // Number of receipts allowed to fetch per request
+ stateCapacity int32 // Number of node data pieces allowed to fetch per request
+
+ blockStarted time.Time // Time instance when the last block (body)fetch was started
+ receiptStarted time.Time // Time instance when the last receipt fetch was started
+ stateStarted time.Time // Time instance when the last node data fetch was started
ignored *set.Set // Set of hashes not to request (didn't have previously)
@@ -68,6 +77,9 @@ type peer struct {
getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position
getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies
+ getReceipts receiptFetcherFn // [eth/63] Method to retrieve a batch of block transaction receipts
+ getNodeData stateFetcherFn // [eth/63] Method to retrieve a batch of state trie data
+
version int // Eth protocol version number to switch strategies
}
@@ -75,12 +87,15 @@ type peer struct {
// mechanisms.
func newPeer(id string, version int, head common.Hash,
getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading
- getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn) *peer {
+ getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn,
+ getReceipts receiptFetcherFn, getNodeData stateFetcherFn) *peer {
return &peer{
- id: id,
- head: head,
- capacity: 1,
- ignored: set.New(),
+ id: id,
+ head: head,
+ blockCapacity: 1,
+ receiptCapacity: 1,
+ stateCapacity: 1,
+ ignored: set.New(),
getRelHashes: getRelHashes,
getAbsHashes: getAbsHashes,
@@ -90,24 +105,34 @@ func newPeer(id string, version int, head common.Hash,
getAbsHeaders: getAbsHeaders,
getBlockBodies: getBlockBodies,
+ getReceipts: getReceipts,
+ getNodeData: getNodeData,
+
version: version,
}
}
// Reset clears the internal state of a peer entity.
func (p *peer) Reset() {
- atomic.StoreInt32(&p.idle, 0)
- atomic.StoreInt32(&p.capacity, 1)
+ atomic.StoreInt32(&p.blockIdle, 0)
+ atomic.StoreInt32(&p.receiptIdle, 0)
+ atomic.StoreInt32(&p.blockCapacity, 1)
+ atomic.StoreInt32(&p.receiptCapacity, 1)
+ atomic.StoreInt32(&p.stateCapacity, 1)
p.ignored.Clear()
}
// Fetch61 sends a block retrieval request to the remote peer.
func (p *peer) Fetch61(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version != 61 {
+ panic(fmt.Sprintf("block fetch [eth/61] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
- if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
+ if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
}
- p.started = time.Now()
+ p.blockStarted = time.Now()
// Convert the hash set to a retrievable slice
hashes := make([]common.Hash, 0, len(request.Hashes))
@@ -119,13 +144,17 @@ func (p *peer) Fetch61(request *fetchRequest) error {
return nil
}
-// Fetch sends a block body retrieval request to the remote peer.
-func (p *peer) Fetch(request *fetchRequest) error {
+// FetchBodies sends a block body retrieval request to the remote peer.
+func (p *peer) FetchBodies(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 62 {
+ panic(fmt.Sprintf("body fetch [eth/62+] requested on eth/%d", p.version))
+ }
// Short circuit if the peer is already fetching
- if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
+ if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) {
return errAlreadyFetching
}
- p.started = time.Now()
+ p.blockStarted = time.Now()
// Convert the header set to a retrievable slice
hashes := make([]common.Hash, 0, len(request.Headers))
@@ -137,55 +166,97 @@ func (p *peer) Fetch(request *fetchRequest) error {
return nil
}
-// SetIdle61 sets the peer to idle, allowing it to execute new retrieval requests.
+// FetchReceipts sends a receipt retrieval request to the remote peer.
+func (p *peer) FetchReceipts(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 63 {
+ panic(fmt.Sprintf("body fetch [eth/63+] requested on eth/%d", p.version))
+ }
+ // Short circuit if the peer is already fetching
+ if !atomic.CompareAndSwapInt32(&p.receiptIdle, 0, 1) {
+ return errAlreadyFetching
+ }
+ p.receiptStarted = time.Now()
+
+ // Convert the header set to a retrievable slice
+ hashes := make([]common.Hash, 0, len(request.Headers))
+ for _, header := range request.Headers {
+ hashes = append(hashes, header.Hash())
+ }
+ go p.getReceipts(hashes)
+
+ return nil
+}
+
+// FetchNodeData sends a node state data retrieval request to the remote peer.
+func (p *peer) FetchNodeData(request *fetchRequest) error {
+ // Sanity check the protocol version
+ if p.version < 63 {
+ panic(fmt.Sprintf("node data fetch [eth/63+] requested on eth/%d", p.version))
+ }
+ // Short circuit if the peer is already fetching
+ if !atomic.CompareAndSwapInt32(&p.stateIdle, 0, 1) {
+ return errAlreadyFetching
+ }
+ p.stateStarted = time.Now()
+
+ // Convert the hash set to a retrievable slice
+ hashes := make([]common.Hash, 0, len(request.Hashes))
+ for hash, _ := range request.Hashes {
+ hashes = append(hashes, hash)
+ }
+ go p.getNodeData(hashes)
+
+ return nil
+}
+
+// SetBlocksIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
-func (p *peer) SetIdle61() {
- // Update the peer's download allowance based on previous performance
- scale := 2.0
- if time.Since(p.started) > blockSoftTTL {
- scale = 0.5
- if time.Since(p.started) > blockHardTTL {
- scale = 1 / float64(MaxBlockFetch) // reduces capacity to 1
- }
- }
- for {
- // Calculate the new download bandwidth allowance
- prev := atomic.LoadInt32(&p.capacity)
- next := int32(math.Max(1, math.Min(float64(MaxBlockFetch), float64(prev)*scale)))
-
- // Try to update the old value
- if atomic.CompareAndSwapInt32(&p.capacity, prev, next) {
- // If we're having problems at 1 capacity, try to find better peers
- if next == 1 {
- p.Demote()
- }
- break
- }
- }
- // Set the peer to idle to allow further block requests
- atomic.StoreInt32(&p.idle, 0)
+// depending on whether the previous fetch completed in time.
+func (p *peer) SetBlocksIdle() {
+ p.setIdle(p.blockStarted, blockSoftTTL, blockHardTTL, MaxBlockFetch, &p.blockCapacity, &p.blockIdle)
}
-// SetIdle sets the peer to idle, allowing it to execute new retrieval requests.
+// SetBodiesIdle sets the peer to idle, allowing it to execute new retrieval requests.
// Its block body retrieval allowance will also be updated either up- or downwards,
-// depending on whether the previous fetch completed in time or not.
-func (p *peer) SetIdle() {
+// depending on whether the previous fetch completed in time.
+func (p *peer) SetBodiesIdle() {
+ p.setIdle(p.blockStarted, bodySoftTTL, bodyHardTTL, MaxBodyFetch, &p.blockCapacity, &p.blockIdle)
+}
+
+// SetReceiptsIdle sets the peer to idle, allowing it to execute new retrieval requests.
+// Its receipt retrieval allowance will also be updated either up- or downwards,
+// depending on whether the previous fetch completed in time.
+func (p *peer) SetReceiptsIdle() {
+ p.setIdle(p.receiptStarted, receiptSoftTTL, receiptHardTTL, MaxReceiptFetch, &p.receiptCapacity, &p.receiptIdle)
+}
+
+// SetNodeDataIdle sets the peer to idle, allowing it to execute new retrieval
+// requests. Its node data retrieval allowance will also be updated either up- or
+// downwards, depending on whether the previous fetch completed in time.
+func (p *peer) SetNodeDataIdle() {
+ p.setIdle(p.stateStarted, stateSoftTTL, stateSoftTTL, MaxStateFetch, &p.stateCapacity, &p.stateIdle)
+}
+
+// setIdle sets the peer to idle, allowing it to execute new retrieval requests.
+// Its data retrieval allowance will also be updated either up- or downwards,
+// depending on whether the previous fetch completed in time.
+func (p *peer) setIdle(started time.Time, softTTL, hardTTL time.Duration, maxFetch int, capacity, idle *int32) {
// Update the peer's download allowance based on previous performance
scale := 2.0
- if time.Since(p.started) > bodySoftTTL {
+ if time.Since(started) > softTTL {
scale = 0.5
- if time.Since(p.started) > bodyHardTTL {
- scale = 1 / float64(MaxBodyFetch) // reduces capacity to 1
+ if time.Since(started) > hardTTL {
+ scale = 1 / float64(maxFetch) // reduces capacity to 1
}
}
for {
// Calculate the new download bandwidth allowance
- prev := atomic.LoadInt32(&p.capacity)
- next := int32(math.Max(1, math.Min(float64(MaxBodyFetch), float64(prev)*scale)))
+ prev := atomic.LoadInt32(capacity)
+ next := int32(math.Max(1, math.Min(float64(maxFetch), float64(prev)*scale)))
// Try to update the old value
- if atomic.CompareAndSwapInt32(&p.capacity, prev, next) {
+ if atomic.CompareAndSwapInt32(capacity, prev, next) {
// If we're having problems at 1 capacity, try to find better peers
if next == 1 {
p.Demote()
@@ -193,14 +264,26 @@ func (p *peer) SetIdle() {
break
}
}
- // Set the peer to idle to allow further block requests
- atomic.StoreInt32(&p.idle, 0)
+ // Set the peer to idle to allow further fetch requests
+ atomic.StoreInt32(idle, 0)
}
-// Capacity retrieves the peers block download allowance based on its previously
-// discovered bandwidth capacity.
-func (p *peer) Capacity() int {
- return int(atomic.LoadInt32(&p.capacity))
+// BlockCapacity retrieves the peers block download allowance based on its
+// previously discovered bandwidth capacity.
+func (p *peer) BlockCapacity() int {
+ return int(atomic.LoadInt32(&p.blockCapacity))
+}
+
+// ReceiptCapacity retrieves the peers block download allowance based on its
+// previously discovered bandwidth capacity.
+func (p *peer) ReceiptCapacity() int {
+ return int(atomic.LoadInt32(&p.receiptCapacity))
+}
+
+// NodeDataCapacity retrieves the peers block download allowance based on its
+// previously discovered bandwidth capacity.
+func (p *peer) NodeDataCapacity() int {
+ return int(atomic.LoadInt32(&p.stateCapacity))
}
// Promote increases the peer's reputation.
@@ -226,7 +309,8 @@ func (p *peer) Demote() {
func (p *peer) String() string {
return fmt.Sprintf("Peer %s [%s]", p.id,
fmt.Sprintf("reputation %3d, ", atomic.LoadInt32(&p.rep))+
- fmt.Sprintf("capacity %3d, ", atomic.LoadInt32(&p.capacity))+
+ fmt.Sprintf("block cap %3d, ", atomic.LoadInt32(&p.blockCapacity))+
+ fmt.Sprintf("receipt cap %3d, ", atomic.LoadInt32(&p.receiptCapacity))+
fmt.Sprintf("ignored %4d", p.ignored.Size()),
)
}
@@ -310,26 +394,63 @@ func (ps *peerSet) AllPeers() []*peer {
return list
}
-// IdlePeers retrieves a flat list of all the currently idle peers within the
+// BlockIdlePeers retrieves a flat list of all the currently idle peers within the
// active peer set, ordered by their reputation.
-func (ps *peerSet) IdlePeers(version int) []*peer {
+func (ps *peerSet) BlockIdlePeers() ([]*peer, int) {
+ idle := func(p *peer) bool {
+ return atomic.LoadInt32(&p.blockIdle) == 0
+ }
+ return ps.idlePeers(61, 61, idle)
+}
+
+// BodyIdlePeers retrieves a flat list of all the currently body-idle peers within
+// the active peer set, ordered by their reputation.
+func (ps *peerSet) BodyIdlePeers() ([]*peer, int) {
+ idle := func(p *peer) bool {
+ return atomic.LoadInt32(&p.blockIdle) == 0
+ }
+ return ps.idlePeers(62, 64, idle)
+}
+
+// ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers
+// within the active peer set, ordered by their reputation.
+func (ps *peerSet) ReceiptIdlePeers() ([]*peer, int) {
+ idle := func(p *peer) bool {
+ return atomic.LoadInt32(&p.receiptIdle) == 0
+ }
+ return ps.idlePeers(63, 64, idle)
+}
+
+// NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle
+// peers within the active peer set, ordered by their reputation.
+func (ps *peerSet) NodeDataIdlePeers() ([]*peer, int) {
+ idle := func(p *peer) bool {
+ return atomic.LoadInt32(&p.stateIdle) == 0
+ }
+ return ps.idlePeers(63, 64, idle)
+}
+
+// idlePeers retrieves a flat list of all currently idle peers satisfying the
+// protocol version constraints, using the provided function to check idleness.
+func (ps *peerSet) idlePeers(minProtocol, maxProtocol int, idleCheck func(*peer) bool) ([]*peer, int) {
ps.lock.RLock()
defer ps.lock.RUnlock()
- list := make([]*peer, 0, len(ps.peers))
+ idle, total := make([]*peer, 0, len(ps.peers)), 0
for _, p := range ps.peers {
- if (version == eth61 && p.version == eth61) || (version >= eth62 && p.version >= eth62) {
- if atomic.LoadInt32(&p.idle) == 0 {
- list = append(list, p)
+ if p.version >= minProtocol && p.version <= maxProtocol {
+ if idleCheck(p) {
+ idle = append(idle, p)
+ }
+ total++
+ }
+ }
+ for i := 0; i < len(idle); i++ {
+ for j := i + 1; j < len(idle); j++ {
+ if atomic.LoadInt32(&idle[i].rep) < atomic.LoadInt32(&idle[j].rep) {
+ idle[i], idle[j] = idle[j], idle[i]
}
}
}
- for i := 0; i < len(list); i++ {
- for j := i + 1; j < len(list); j++ {
- if atomic.LoadInt32(&list[i].rep) < atomic.LoadInt32(&list[j].rep) {
- list[i], list[j] = list[j], list[i]
- }
- }
- }
- return list
+ return idle, total
}
diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go
index 49d1046fb..56b46e285 100644
--- a/eth/downloader/queue.go
+++ b/eth/downloader/queue.go
@@ -23,25 +23,32 @@ import (
"errors"
"fmt"
"sync"
+ "sync/atomic"
"time"
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/crypto"
+ "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
+ "github.com/ethereum/go-ethereum/trie"
+ "github.com/rcrowley/go-metrics"
"gopkg.in/karalabe/cookiejar.v2/collections/prque"
)
var (
- blockCacheLimit = 8 * MaxBlockFetch // Maximum number of blocks to cache before throttling the download
+ blockCacheLimit = 1024 // Maximum number of blocks to cache before throttling the download
)
var (
errNoFetchesPending = errors.New("no fetches pending")
+ errStateSyncPending = errors.New("state trie sync already scheduled")
errStaleDelivery = errors.New("stale delivery")
)
-// fetchRequest is a currently running block retrieval operation.
+// fetchRequest is a currently running data retrieval operation.
type fetchRequest struct {
Peer *peer // Peer to which the request was sent
Hashes map[common.Hash]int // [eth/61] Requested hashes with their insertion index (priority)
@@ -49,35 +56,72 @@ type fetchRequest struct {
Time time.Time // Time when the request was made
}
+// fetchResult is a struct collecting partial results from data fetchers until
+// all outstanding pieces complete and the result as a whole can be processed.
+type fetchResult struct {
+ Pending int // Number of data fetches still pending
+
+ Header *types.Header
+ Uncles []*types.Header
+ Transactions types.Transactions
+ Receipts types.Receipts
+}
+
// queue represents hashes that are either need fetching or are being fetched
type queue struct {
+ mode SyncMode // Synchronisation mode to decide on the block parts to schedule for fetching
+ fastSyncPivot uint64 // Block number where the fast sync pivots into archive synchronisation mode
+
hashPool map[common.Hash]int // [eth/61] Pending hashes, mapping to their insertion index (priority)
hashQueue *prque.Prque // [eth/61] Priority queue of the block hashes to fetch
hashCounter int // [eth/61] Counter indexing the added hashes to ensure retrieval order
- headerPool map[common.Hash]*types.Header // [eth/62] Pending headers, mapping from their hashes
- headerQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the bodies for
- headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
+ headerHead common.Hash // [eth/62] Hash of the last queued header to verify order
- pendPool map[string]*fetchRequest // Currently pending block retrieval operations
+ blockTaskPool map[common.Hash]*types.Header // [eth/62] Pending block (body) retrieval tasks, mapping hashes to headers
+ blockTaskQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the blocks (bodies) for
+ blockPendPool map[string]*fetchRequest // [eth/62] Currently pending block (body) retrieval operations
+ blockDonePool map[common.Hash]struct{} // [eth/62] Set of the completed block (body) fetches
- blockPool map[common.Hash]uint64 // Hash-set of the downloaded data blocks, mapping to cache indexes
- blockCache []*Block // Downloaded but not yet delivered blocks
- blockOffset uint64 // Offset of the first cached block in the block-chain
+ receiptTaskPool map[common.Hash]*types.Header // [eth/63] Pending receipt retrieval tasks, mapping hashes to headers
+ receiptTaskQueue *prque.Prque // [eth/63] Priority queue of the headers to fetch the receipts for
+ receiptPendPool map[string]*fetchRequest // [eth/63] Currently pending receipt retrieval operations
+ receiptDonePool map[common.Hash]struct{} // [eth/63] Set of the completed receipt fetches
+
+ stateTaskIndex int // [eth/63] Counter indexing the added hashes to ensure prioritised retrieval order
+ stateTaskPool map[common.Hash]int // [eth/63] Pending node data retrieval tasks, mapping to their priority
+ stateTaskQueue *prque.Prque // [eth/63] Priority queue of the hashes to fetch the node data for
+ statePendPool map[string]*fetchRequest // [eth/63] Currently pending node data retrieval operations
+
+ stateDatabase ethdb.Database // [eth/63] Trie database to populate during state reassembly
+ stateScheduler *state.StateSync // [eth/63] State trie synchronisation scheduler and integrator
+ stateProcessors int32 // [eth/63] Number of currently running state processors
+ stateSchedLock sync.RWMutex // [eth/63] Lock serialising access to the state scheduler
+
+ resultCache []*fetchResult // Downloaded but not yet delivered fetch results
+ resultOffset uint64 // Offset of the first cached fetch result in the block chain
lock sync.RWMutex
}
// newQueue creates a new download queue for scheduling block retrieval.
-func newQueue() *queue {
+func newQueue(stateDb ethdb.Database) *queue {
return &queue{
- hashPool: make(map[common.Hash]int),
- hashQueue: prque.New(),
- headerPool: make(map[common.Hash]*types.Header),
- headerQueue: prque.New(),
- pendPool: make(map[string]*fetchRequest),
- blockPool: make(map[common.Hash]uint64),
- blockCache: make([]*Block, blockCacheLimit),
+ hashPool: make(map[common.Hash]int),
+ hashQueue: prque.New(),
+ blockTaskPool: make(map[common.Hash]*types.Header),
+ blockTaskQueue: prque.New(),
+ blockPendPool: make(map[string]*fetchRequest),
+ blockDonePool: make(map[common.Hash]struct{}),
+ receiptTaskPool: make(map[common.Hash]*types.Header),
+ receiptTaskQueue: prque.New(),
+ receiptPendPool: make(map[string]*fetchRequest),
+ receiptDonePool: make(map[common.Hash]struct{}),
+ stateTaskPool: make(map[common.Hash]int),
+ stateTaskQueue: prque.New(),
+ statePendPool: make(map[string]*fetchRequest),
+ stateDatabase: stateDb,
+ resultCache: make([]*fetchResult, blockCacheLimit),
}
}
@@ -86,85 +130,156 @@ func (q *queue) Reset() {
q.lock.Lock()
defer q.lock.Unlock()
+ q.stateSchedLock.Lock()
+ defer q.stateSchedLock.Unlock()
+
+ q.mode = FullSync
+ q.fastSyncPivot = 0
+
q.hashPool = make(map[common.Hash]int)
q.hashQueue.Reset()
q.hashCounter = 0
- q.headerPool = make(map[common.Hash]*types.Header)
- q.headerQueue.Reset()
q.headerHead = common.Hash{}
- q.pendPool = make(map[string]*fetchRequest)
+ q.blockTaskPool = make(map[common.Hash]*types.Header)
+ q.blockTaskQueue.Reset()
+ q.blockPendPool = make(map[string]*fetchRequest)
+ q.blockDonePool = make(map[common.Hash]struct{})
- q.blockPool = make(map[common.Hash]uint64)
- q.blockOffset = 0
- q.blockCache = make([]*Block, blockCacheLimit)
+ q.receiptTaskPool = make(map[common.Hash]*types.Header)
+ q.receiptTaskQueue.Reset()
+ q.receiptPendPool = make(map[string]*fetchRequest)
+ q.receiptDonePool = make(map[common.Hash]struct{})
+
+ q.stateTaskIndex = 0
+ q.stateTaskPool = make(map[common.Hash]int)
+ q.stateTaskQueue.Reset()
+ q.statePendPool = make(map[string]*fetchRequest)
+ q.stateScheduler = nil
+
+ q.resultCache = make([]*fetchResult, blockCacheLimit)
+ q.resultOffset = 0
}
-// Size retrieves the number of blocks in the queue, returning separately for
-// pending and already downloaded.
-func (q *queue) Size() (int, int) {
+// PendingBlocks retrieves the number of block (body) requests pending for retrieval.
+func (q *queue) PendingBlocks() int {
q.lock.RLock()
defer q.lock.RUnlock()
- return len(q.hashPool) + len(q.headerPool), len(q.blockPool)
+ return q.hashQueue.Size() + q.blockTaskQueue.Size()
}
-// Pending retrieves the number of blocks pending for retrieval.
-func (q *queue) Pending() int {
+// PendingReceipts retrieves the number of block receipts pending for retrieval.
+func (q *queue) PendingReceipts() int {
q.lock.RLock()
defer q.lock.RUnlock()
- return q.hashQueue.Size() + q.headerQueue.Size()
+ return q.receiptTaskQueue.Size()
}
-// InFlight retrieves the number of fetch requests currently in flight.
-func (q *queue) InFlight() int {
+// PendingNodeData retrieves the number of node data entries pending for retrieval.
+func (q *queue) PendingNodeData() int {
+ q.stateSchedLock.RLock()
+ defer q.stateSchedLock.RUnlock()
+
+ if q.stateScheduler != nil {
+ return q.stateScheduler.Pending()
+ }
+ return 0
+}
+
+// InFlightBlocks retrieves whether there are block fetch requests currently in
+// flight.
+func (q *queue) InFlightBlocks() bool {
q.lock.RLock()
defer q.lock.RUnlock()
- return len(q.pendPool)
+ return len(q.blockPendPool) > 0
}
-// Throttle checks if the download should be throttled (active block fetches
-// exceed block cache).
-func (q *queue) Throttle() bool {
+// InFlightReceipts retrieves whether there are receipt fetch requests currently
+// in flight.
+func (q *queue) InFlightReceipts() bool {
q.lock.RLock()
defer q.lock.RUnlock()
- // Calculate the currently in-flight block requests
+ return len(q.receiptPendPool) > 0
+}
+
+// InFlightNodeData retrieves whether there are node data entry fetch requests
+// currently in flight.
+func (q *queue) InFlightNodeData() bool {
+ q.lock.RLock()
+ defer q.lock.RUnlock()
+
+ return len(q.statePendPool)+int(atomic.LoadInt32(&q.stateProcessors)) > 0
+}
+
+// Idle returns if the queue is fully idle or has some data still inside. This
+// method is used by the tester to detect termination events.
+func (q *queue) Idle() bool {
+ q.lock.RLock()
+ defer q.lock.RUnlock()
+
+ queued := q.hashQueue.Size() + q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + q.stateTaskQueue.Size()
+ pending := len(q.blockPendPool) + len(q.receiptPendPool) + len(q.statePendPool)
+ cached := len(q.blockDonePool) + len(q.receiptDonePool)
+
+ q.stateSchedLock.RLock()
+ if q.stateScheduler != nil {
+ queued += q.stateScheduler.Pending()
+ }
+ q.stateSchedLock.RUnlock()
+
+ return (queued + pending + cached) == 0
+}
+
+// FastSyncPivot retrieves the currently used fast sync pivot point.
+func (q *queue) FastSyncPivot() uint64 {
+ q.lock.RLock()
+ defer q.lock.RUnlock()
+
+ return q.fastSyncPivot
+}
+
+// ShouldThrottleBlocks checks if the download should be throttled (active block (body)
+// fetches exceed block cache).
+func (q *queue) ShouldThrottleBlocks() bool {
+ q.lock.RLock()
+ defer q.lock.RUnlock()
+
+ // Calculate the currently in-flight block (body) requests
pending := 0
- for _, request := range q.pendPool {
+ for _, request := range q.blockPendPool {
pending += len(request.Hashes) + len(request.Headers)
}
- // Throttle if more blocks are in-flight than free space in the cache
- return pending >= len(q.blockCache)-len(q.blockPool)
+ // Throttle if more blocks (bodies) are in-flight than free space in the cache
+ return pending >= len(q.resultCache)-len(q.blockDonePool)
}
-// Has checks if a hash is within the download queue or not.
-func (q *queue) Has(hash common.Hash) bool {
+// ShouldThrottleReceipts checks if the download should be throttled (active receipt
+// fetches exceed block cache).
+func (q *queue) ShouldThrottleReceipts() bool {
q.lock.RLock()
defer q.lock.RUnlock()
- if _, ok := q.hashPool[hash]; ok {
- return true
+ // Calculate the currently in-flight receipt requests
+ pending := 0
+ for _, request := range q.receiptPendPool {
+ pending += len(request.Headers)
}
- if _, ok := q.headerPool[hash]; ok {
- return true
- }
- if _, ok := q.blockPool[hash]; ok {
- return true
- }
- return false
+ // Throttle if more receipts are in-flight than free space in the cache
+ return pending >= len(q.resultCache)-len(q.receiptDonePool)
}
-// Insert61 adds a set of hashes for the download queue for scheduling, returning
+// Schedule61 adds a set of hashes for the download queue for scheduling, returning
// the new hashes encountered.
-func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
+func (q *queue) Schedule61(hashes []common.Hash, fifo bool) []common.Hash {
q.lock.Lock()
defer q.lock.Unlock()
- // Insert all the hashes prioritized in the arrival order
+ // Insert all the hashes prioritised in the arrival order
inserts := make([]common.Hash, 0, len(hashes))
for _, hash := range hashes {
// Skip anything we already have
@@ -186,22 +301,17 @@ func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
return inserts
}
-// Insert adds a set of headers for the download queue for scheduling, returning
+// Schedule adds a set of headers for the download queue for scheduling, returning
// the new headers encountered.
-func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header {
+func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header {
q.lock.Lock()
defer q.lock.Unlock()
- // Insert all the headers prioritized by the contained block number
+ // Insert all the headers prioritised by the contained block number
inserts := make([]*types.Header, 0, len(headers))
for _, header := range headers {
- // Make sure no duplicate requests are executed
+ // Make sure chain order is honoured and preserved throughout
hash := header.Hash()
- if _, ok := q.headerPool[hash]; ok {
- glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled", header.Number.Uint64(), hash[:4])
- continue
- }
- // Make sure chain order is honored and preserved throughout
if header.Number == nil || header.Number.Uint64() != from {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ordering, expected %d", header.Number, hash[:4], from)
break
@@ -210,96 +320,187 @@ func (q *queue) Insert(headers []*types.Header, from uint64) []*types.Header {
glog.V(logger.Warn).Infof("Header #%v [%x] broke chain ancestry", header.Number, hash[:4])
break
}
- // Queue the header for body retrieval
+ // Make sure no duplicate requests are executed
+ if _, ok := q.blockTaskPool[hash]; ok {
+ glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled for block fetch", header.Number.Uint64(), hash[:4])
+ continue
+ }
+ if _, ok := q.receiptTaskPool[hash]; ok {
+ glog.V(logger.Warn).Infof("Header #%d [%x] already scheduled for receipt fetch", header.Number.Uint64(), hash[:4])
+ continue
+ }
+ // Queue the header for content retrieval
+ q.blockTaskPool[hash] = header
+ q.blockTaskQueue.Push(header, -float32(header.Number.Uint64()))
+
+ if q.mode == FastSync && header.Number.Uint64() <= q.fastSyncPivot {
+ // Fast phase of the fast sync, retrieve receipts too
+ q.receiptTaskPool[hash] = header
+ q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64()))
+ }
+ if q.mode == FastSync && header.Number.Uint64() == q.fastSyncPivot {
+ // Pivoting point of the fast sync, retrieve the state tries
+ q.stateSchedLock.Lock()
+ q.stateScheduler = state.NewStateSync(header.Root, q.stateDatabase)
+ q.stateSchedLock.Unlock()
+ }
inserts = append(inserts, header)
- q.headerPool[hash] = header
- q.headerQueue.Push(header, -float32(header.Number.Uint64()))
q.headerHead = hash
from++
}
return inserts
}
-// GetHeadBlock retrieves the first block from the cache, or nil if it hasn't
+// GetHeadResult retrieves the first fetch result from the cache, or nil if it hasn't
// been downloaded yet (or simply non existent).
-func (q *queue) GetHeadBlock() *Block {
+func (q *queue) GetHeadResult() *fetchResult {
q.lock.RLock()
defer q.lock.RUnlock()
- if len(q.blockCache) == 0 {
+ // If there are no results pending, return nil
+ if len(q.resultCache) == 0 || q.resultCache[0] == nil {
return nil
}
- return q.blockCache[0]
-}
-
-// GetBlock retrieves a downloaded block, or nil if non-existent.
-func (q *queue) GetBlock(hash common.Hash) *Block {
- q.lock.RLock()
- defer q.lock.RUnlock()
-
- // Short circuit if the block hasn't been downloaded yet
- index, ok := q.blockPool[hash]
- if !ok {
+ // If the next result is still incomplete, return nil
+ if q.resultCache[0].Pending > 0 {
return nil
}
- // Return the block if it's still available in the cache
- if q.blockOffset <= index && index < q.blockOffset+uint64(len(q.blockCache)) {
- return q.blockCache[index-q.blockOffset]
+ // If the next result is the fast sync pivot...
+ if q.mode == FastSync && q.resultCache[0].Header.Number.Uint64() == q.fastSyncPivot {
+ // If the pivot state trie is still being pulled, return nil
+ if len(q.stateTaskPool) > 0 {
+ return nil
+ }
+ if q.PendingNodeData() > 0 {
+ return nil
+ }
+ // If the state is done, but not enough post-pivot headers were verified, stall...
+ for i := 0; i < fsHeaderForceVerify; i++ {
+ if i+1 >= len(q.resultCache) || q.resultCache[i+1] == nil {
+ return nil
+ }
+ }
}
- return nil
+ return q.resultCache[0]
}
-// TakeBlocks retrieves and permanently removes a batch of blocks from the cache.
-func (q *queue) TakeBlocks() []*Block {
+// TakeResults retrieves and permanently removes a batch of fetch results from
+// the cache.
+func (q *queue) TakeResults() []*fetchResult {
q.lock.Lock()
defer q.lock.Unlock()
- // Accumulate all available blocks
- blocks := []*Block{}
- for _, block := range q.blockCache {
- if block == nil {
+ // Accumulate all available results
+ results := []*fetchResult{}
+ for i, result := range q.resultCache {
+ // Stop if no more results are ready
+ if result == nil || result.Pending > 0 {
break
}
- blocks = append(blocks, block)
- delete(q.blockPool, block.RawBlock.Hash())
- }
- // Delete the blocks from the slice and let them be garbage collected
- // without this slice trick the blocks would stay in memory until nil
- // would be assigned to q.blocks
- copy(q.blockCache, q.blockCache[len(blocks):])
- for k, n := len(q.blockCache)-len(blocks), len(q.blockCache); k < n; k++ {
- q.blockCache[k] = nil
- }
- q.blockOffset += uint64(len(blocks))
+ // The fast sync pivot block may only be processed after state fetch completes
+ if q.mode == FastSync && result.Header.Number.Uint64() == q.fastSyncPivot {
+ if len(q.stateTaskPool) > 0 {
+ break
+ }
+ if q.PendingNodeData() > 0 {
+ break
+ }
+ // Even is state fetch is done, ensure post-pivot headers passed verifications
+ safe := true
+ for j := 0; j < fsHeaderForceVerify; j++ {
+ if i+j+1 >= len(q.resultCache) || q.resultCache[i+j+1] == nil {
+ safe = false
+ }
+ }
+ if !safe {
+ break
+ }
+ }
+ // If we've just inserted the fast sync pivot, stop as the following batch needs different insertion
+ if q.mode == FastSync && result.Header.Number.Uint64() == q.fastSyncPivot+1 && len(results) > 0 {
+ break
+ }
+ results = append(results, result)
- return blocks
+ hash := result.Header.Hash()
+ delete(q.blockDonePool, hash)
+ delete(q.receiptDonePool, hash)
+ }
+ // Delete the results from the slice and let them be garbage collected
+ // without this slice trick the results would stay in memory until nil
+ // would be assigned to them.
+ copy(q.resultCache, q.resultCache[len(results):])
+ for k, n := len(q.resultCache)-len(results), len(q.resultCache); k < n; k++ {
+ q.resultCache[k] = nil
+ }
+ q.resultOffset += uint64(len(results))
+
+ return results
}
-// Reserve61 reserves a set of hashes for the given peer, skipping any previously
-// failed download.
-func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
+// ReserveBlocks reserves a set of block hashes for the given peer, skipping any
+// previously failed download.
+func (q *queue) ReserveBlocks(p *peer, count int) *fetchRequest {
q.lock.Lock()
defer q.lock.Unlock()
- // Short circuit if the pool has been depleted, or if the peer's already
- // downloading something (sanity check not to corrupt state)
- if q.hashQueue.Empty() {
- return nil
+ return q.reserveHashes(p, count, q.hashQueue, nil, q.blockPendPool, len(q.resultCache)-len(q.blockDonePool))
+}
+
+// ReserveNodeData reserves a set of node data hashes for the given peer, skipping
+// any previously failed download.
+func (q *queue) ReserveNodeData(p *peer, count int) *fetchRequest {
+ // Create a task generator to fetch status-fetch tasks if all schedules ones are done
+ generator := func(max int) {
+ q.stateSchedLock.Lock()
+ defer q.stateSchedLock.Unlock()
+
+ if q.stateScheduler != nil {
+ for _, hash := range q.stateScheduler.Missing(max) {
+ q.stateTaskPool[hash] = q.stateTaskIndex
+ q.stateTaskQueue.Push(hash, -float32(q.stateTaskIndex))
+ q.stateTaskIndex++
+ }
+ }
}
- if _, ok := q.pendPool[p.id]; ok {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.reserveHashes(p, count, q.stateTaskQueue, generator, q.statePendPool, count)
+}
+
+// reserveHashes reserves a set of hashes for the given peer, skipping previously
+// failed ones.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
+func (q *queue) reserveHashes(p *peer, count int, taskQueue *prque.Prque, taskGen func(int), pendPool map[string]*fetchRequest, maxPending int) *fetchRequest {
+ // Short circuit if the peer's already downloading something (sanity check to
+ // not corrupt state)
+ if _, ok := pendPool[p.id]; ok {
return nil
}
// Calculate an upper limit on the hashes we might fetch (i.e. throttling)
- space := len(q.blockCache) - len(q.blockPool)
- for _, request := range q.pendPool {
- space -= len(request.Hashes)
+ allowance := maxPending
+ if allowance > 0 {
+ for _, request := range pendPool {
+ allowance -= len(request.Hashes)
+ }
+ }
+ // If there's a task generator, ask it to fill our task queue
+ if taskGen != nil && taskQueue.Size() < allowance {
+ taskGen(allowance - taskQueue.Size())
+ }
+ if taskQueue.Empty() {
+ return nil
}
// Retrieve a batch of hashes, skipping previously failed ones
send := make(map[common.Hash]int)
skip := make(map[common.Hash]int)
- for proc := 0; proc < space && len(send) < count && !q.hashQueue.Empty(); proc++ {
- hash, priority := q.hashQueue.Pop()
+ for proc := 0; (allowance == 0 || proc < allowance) && len(send) < count && !taskQueue.Empty(); proc++ {
+ hash, priority := taskQueue.Pop()
if p.ignored.Has(hash) {
skip[hash.(common.Hash)] = int(priority)
} else {
@@ -308,7 +509,7 @@ func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
}
// Merge all the skipped hashes back
for hash, index := range skip {
- q.hashQueue.Push(hash, float32(index))
+ taskQueue.Push(hash, float32(index))
}
// Assemble and return the block download request
if len(send) == 0 {
@@ -319,49 +520,93 @@ func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
Hashes: send,
Time: time.Now(),
}
- q.pendPool[p.id] = request
+ pendPool[p.id] = request
return request
}
-// Reserve reserves a set of headers for the given peer, skipping any previously
-// failed download. Beside the next batch of needed fetches, it also returns a
-// flag whether empty blocks were queued requiring processing.
-func (q *queue) Reserve(p *peer, count int) (*fetchRequest, bool, error) {
+// ReserveBodies reserves a set of body fetches for the given peer, skipping any
+// previously failed downloads. Beside the next batch of needed fetches, it also
+// returns a flag whether empty blocks were queued requiring processing.
+func (q *queue) ReserveBodies(p *peer, count int) (*fetchRequest, bool, error) {
+ isNoop := func(header *types.Header) bool {
+ return header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash
+ }
q.lock.Lock()
defer q.lock.Unlock()
+ return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, isNoop)
+}
+
+// ReserveReceipts reserves a set of receipt fetches for the given peer, skipping
+// any previously failed downloads. Beside the next batch of needed fetches, it
+// also returns a flag whether empty receipts were queued requiring importing.
+func (q *queue) ReserveReceipts(p *peer, count int) (*fetchRequest, bool, error) {
+ isNoop := func(header *types.Header) bool {
+ return header.ReceiptHash == types.EmptyRootHash
+ }
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, isNoop)
+}
+
+// reserveHeaders reserves a set of data download operations for a given peer,
+// skipping any previously failed ones. This method is a generic version used
+// by the individual special reservation functions.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
+func (q *queue) reserveHeaders(p *peer, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque,
+ pendPool map[string]*fetchRequest, donePool map[common.Hash]struct{}, isNoop func(*types.Header) bool) (*fetchRequest, bool, error) {
// Short circuit if the pool has been depleted, or if the peer's already
// downloading something (sanity check not to corrupt state)
- if q.headerQueue.Empty() {
+ if taskQueue.Empty() {
return nil, false, nil
}
- if _, ok := q.pendPool[p.id]; ok {
+ if _, ok := pendPool[p.id]; ok {
return nil, false, nil
}
- // Calculate an upper limit on the bodies we might fetch (i.e. throttling)
- space := len(q.blockCache) - len(q.blockPool)
- for _, request := range q.pendPool {
+ // Calculate an upper limit on the items we might fetch (i.e. throttling)
+ space := len(q.resultCache) - len(donePool)
+ for _, request := range pendPool {
space -= len(request.Headers)
}
- // Retrieve a batch of headers, skipping previously failed ones
+ // Retrieve a batch of tasks, skipping previously failed ones
send := make([]*types.Header, 0, count)
skip := make([]*types.Header, 0)
- process := false
- for proc := 0; proc < space && len(send) < count && !q.headerQueue.Empty(); proc++ {
- header := q.headerQueue.PopItem().(*types.Header)
+ progress := false
+ for proc := 0; proc < space && len(send) < count && !taskQueue.Empty(); proc++ {
+ header := taskQueue.PopItem().(*types.Header)
- // If the header defines an empty block, deliver straight
- if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
- if err := q.enqueue("", types.NewBlockWithHeader(header)); err != nil {
- return nil, false, errInvalidChain
+ // If we're the first to request this task, initialise the result container
+ index := int(header.Number.Int64() - int64(q.resultOffset))
+ if index >= len(q.resultCache) || index < 0 {
+ return nil, false, errInvalidChain
+ }
+ if q.resultCache[index] == nil {
+ components := 1
+ if q.mode == FastSync && header.Number.Uint64() <= q.fastSyncPivot {
+ components = 2
}
- delete(q.headerPool, header.Hash())
- process, space, proc = true, space-1, proc-1
+ q.resultCache[index] = &fetchResult{
+ Pending: components,
+ Header: header,
+ }
+ }
+ // If this fetch task is a noop, skip this fetch operation
+ if isNoop(header) {
+ donePool[header.Hash()] = struct{}{}
+ delete(taskPool, header.Hash())
+
+ space, proc = space-1, proc-1
+ q.resultCache[index].Pending--
+ progress = true
continue
}
- // If it's a content block, add to the body fetch request
+ // Otherwise unless the peer is known not to have the data, add to the retrieve list
if p.ignored.Has(header.Hash()) {
skip = append(skip, header)
} else {
@@ -370,81 +615,168 @@ func (q *queue) Reserve(p *peer, count int) (*fetchRequest, bool, error) {
}
// Merge all the skipped headers back
for _, header := range skip {
- q.headerQueue.Push(header, -float32(header.Number.Uint64()))
+ taskQueue.Push(header, -float32(header.Number.Uint64()))
}
// Assemble and return the block download request
if len(send) == 0 {
- return nil, process, nil
+ return nil, progress, nil
}
request := &fetchRequest{
Peer: p,
Headers: send,
Time: time.Now(),
}
- q.pendPool[p.id] = request
+ pendPool[p.id] = request
- return request, process, nil
+ return request, progress, nil
}
-// Cancel aborts a fetch request, returning all pending hashes to the queue.
-func (q *queue) Cancel(request *fetchRequest) {
+// CancelBlocks aborts a fetch request, returning all pending hashes to the queue.
+func (q *queue) CancelBlocks(request *fetchRequest) {
+ q.cancel(request, q.hashQueue, q.blockPendPool)
+}
+
+// CancelBodies aborts a body fetch request, returning all pending headers to the
+// task queue.
+func (q *queue) CancelBodies(request *fetchRequest) {
+ q.cancel(request, q.blockTaskQueue, q.blockPendPool)
+}
+
+// CancelReceipts aborts a body fetch request, returning all pending headers to
+// the task queue.
+func (q *queue) CancelReceipts(request *fetchRequest) {
+ q.cancel(request, q.receiptTaskQueue, q.receiptPendPool)
+}
+
+// CancelNodeData aborts a node state data fetch request, returning all pending
+// hashes to the task queue.
+func (q *queue) CancelNodeData(request *fetchRequest) {
+ q.cancel(request, q.stateTaskQueue, q.statePendPool)
+}
+
+// Cancel aborts a fetch request, returning all pending hashes to the task queue.
+func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest) {
q.lock.Lock()
defer q.lock.Unlock()
for hash, index := range request.Hashes {
- q.hashQueue.Push(hash, float32(index))
+ taskQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
- q.headerQueue.Push(header, -float32(header.Number.Uint64()))
+ taskQueue.Push(header, -float32(header.Number.Uint64()))
}
- delete(q.pendPool, request.Peer.id)
+ delete(pendPool, request.Peer.id)
}
-// Expire checks for in flight requests that exceeded a timeout allowance,
-// canceling them and returning the responsible peers for penalization.
-func (q *queue) Expire(timeout time.Duration) []string {
+// Revoke cancels all pending requests belonging to a given peer. This method is
+// meant to be called during a peer drop to quickly reassign owned data fetches
+// to remaining nodes.
+func (q *queue) Revoke(peerId string) {
q.lock.Lock()
defer q.lock.Unlock()
+ if request, ok := q.blockPendPool[peerId]; ok {
+ for hash, index := range request.Hashes {
+ q.hashQueue.Push(hash, float32(index))
+ }
+ for _, header := range request.Headers {
+ q.blockTaskQueue.Push(header, -float32(header.Number.Uint64()))
+ }
+ delete(q.blockPendPool, peerId)
+ }
+ if request, ok := q.receiptPendPool[peerId]; ok {
+ for _, header := range request.Headers {
+ q.receiptTaskQueue.Push(header, -float32(header.Number.Uint64()))
+ }
+ delete(q.receiptPendPool, peerId)
+ }
+ if request, ok := q.statePendPool[peerId]; ok {
+ for hash, index := range request.Hashes {
+ q.stateTaskQueue.Push(hash, float32(index))
+ }
+ delete(q.statePendPool, peerId)
+ }
+}
+
+// ExpireBlocks checks for in flight requests that exceeded a timeout allowance,
+// canceling them and returning the responsible peers for penalisation.
+func (q *queue) ExpireBlocks(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.expire(timeout, q.blockPendPool, q.hashQueue, blockTimeoutMeter)
+}
+
+// ExpireBodies checks for in flight block body requests that exceeded a timeout
+// allowance, canceling them and returning the responsible peers for penalisation.
+func (q *queue) ExpireBodies(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.expire(timeout, q.blockPendPool, q.blockTaskQueue, bodyTimeoutMeter)
+}
+
+// ExpireReceipts checks for in flight receipt requests that exceeded a timeout
+// allowance, canceling them and returning the responsible peers for penalisation.
+func (q *queue) ExpireReceipts(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue, receiptTimeoutMeter)
+}
+
+// ExpireNodeData checks for in flight node data requests that exceeded a timeout
+// allowance, canceling them and returning the responsible peers for penalisation.
+func (q *queue) ExpireNodeData(timeout time.Duration) []string {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ return q.expire(timeout, q.statePendPool, q.stateTaskQueue, stateTimeoutMeter)
+}
+
+// expire is the generic check that move expired tasks from a pending pool back
+// into a task pool, returning all entities caught with expired tasks.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
+func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque, timeoutMeter metrics.Meter) []string {
// Iterate over the expired requests and return each to the queue
peers := []string{}
- for id, request := range q.pendPool {
+ for id, request := range pendPool {
if time.Since(request.Time) > timeout {
// Update the metrics with the timeout
- if len(request.Hashes) > 0 {
- blockTimeoutMeter.Mark(1)
- } else {
- bodyTimeoutMeter.Mark(1)
- }
+ timeoutMeter.Mark(1)
+
// Return any non satisfied requests to the pool
for hash, index := range request.Hashes {
- q.hashQueue.Push(hash, float32(index))
+ taskQueue.Push(hash, float32(index))
}
for _, header := range request.Headers {
- q.headerQueue.Push(header, -float32(header.Number.Uint64()))
+ taskQueue.Push(header, -float32(header.Number.Uint64()))
}
peers = append(peers, id)
}
}
// Remove the expired requests from the pending pool
for _, id := range peers {
- delete(q.pendPool, id)
+ delete(pendPool, id)
}
return peers
}
-// Deliver61 injects a block retrieval response into the download queue.
-func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
+// DeliverBlocks injects a block retrieval response into the download queue.
+func (q *queue) DeliverBlocks(id string, blocks []*types.Block) error {
q.lock.Lock()
defer q.lock.Unlock()
// Short circuit if the blocks were never requested
- request := q.pendPool[id]
+ request := q.blockPendPool[id]
if request == nil {
return errNoFetchesPending
}
blockReqTimer.UpdateSince(request.Time)
- delete(q.pendPool, id)
+ delete(q.blockPendPool, id)
// If no blocks were retrieved, mark them as unavailable for the origin peer
if len(blocks) == 0 {
@@ -461,10 +793,19 @@ func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
errs = append(errs, fmt.Errorf("non-requested block %x", hash))
continue
}
- // Queue the block up for processing
- if err := q.enqueue(id, block); err != nil {
- return err
+ // Reconstruct the next result if contents match up
+ index := int(block.Number().Int64() - int64(q.resultOffset))
+ if index >= len(q.resultCache) || index < 0 {
+ errs = []error{errInvalidChain}
+ break
}
+ q.resultCache[index] = &fetchResult{
+ Header: block.Header(),
+ Transactions: block.Transactions(),
+ Uncles: block.Uncles(),
+ }
+ q.blockDonePool[block.Hash()] = struct{}{}
+
delete(request.Hashes, hash)
delete(q.hashPool, hash)
}
@@ -473,74 +814,14 @@ func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
q.hashQueue.Push(hash, float32(index))
}
// If none of the blocks were good, it's a stale delivery
- if len(errs) != 0 {
- if len(errs) == len(blocks) {
- return errStaleDelivery
- }
- return fmt.Errorf("multiple failures: %v", errs)
- }
- return nil
-}
-
-// Deliver injects a block body retrieval response into the download queue.
-func (q *queue) Deliver(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
- q.lock.Lock()
- defer q.lock.Unlock()
-
- // Short circuit if the block bodies were never requested
- request := q.pendPool[id]
- if request == nil {
- return errNoFetchesPending
- }
- bodyReqTimer.UpdateSince(request.Time)
- delete(q.pendPool, id)
-
- // If no block bodies were retrieved, mark them as unavailable for the origin peer
- if len(txLists) == 0 || len(uncleLists) == 0 {
- for hash, _ := range request.Headers {
- request.Peer.ignored.Add(hash)
- }
- }
- // Assemble each of the block bodies with their headers and queue for processing
- errs := make([]error, 0)
- for i, header := range request.Headers {
- // Short circuit block assembly if no more bodies are found
- if i >= len(txLists) || i >= len(uncleLists) {
- break
- }
- // Reconstruct the next block if contents match up
- if types.DeriveSha(types.Transactions(txLists[i])) != header.TxHash || types.CalcUncleHash(uncleLists[i]) != header.UncleHash {
- errs = []error{errInvalidBody}
- break
- }
- block := types.NewBlockWithHeader(header).WithBody(txLists[i], uncleLists[i])
-
- // Queue the block up for processing
- if err := q.enqueue(id, block); err != nil {
- errs = []error{err}
- break
- }
- request.Headers[i] = nil
- delete(q.headerPool, header.Hash())
- }
- // Return all failed or missing fetches to the queue
- for _, header := range request.Headers {
- if header != nil {
- q.headerQueue.Push(header, -float32(header.Number.Uint64()))
- }
- }
- // If none of the blocks were good, it's a stale delivery
switch {
case len(errs) == 0:
return nil
- case len(errs) == 1 && errs[0] == errInvalidBody:
- return errInvalidBody
+ case len(errs) == 1 && (errs[0] == errInvalidChain || errs[0] == errInvalidBlock):
+ return errs[0]
- case len(errs) == 1 && errs[0] == errInvalidChain:
- return errInvalidChain
-
- case len(errs) == len(request.Headers):
+ case len(errs) == len(blocks):
return errStaleDelivery
default:
@@ -548,29 +829,197 @@ func (q *queue) Deliver(id string, txLists [][]*types.Transaction, uncleLists []
}
}
-// enqueue inserts a new block into the final delivery queue, waiting for pickup
-// by the processor.
-func (q *queue) enqueue(origin string, block *types.Block) error {
- // If a requested block falls out of the range, the hash chain is invalid
- index := int(int64(block.NumberU64()) - int64(q.blockOffset))
- if index >= len(q.blockCache) || index < 0 {
- return errInvalidChain
- }
- // Otherwise merge the block and mark the hash done
- q.blockCache[index] = &Block{
- RawBlock: block,
- OriginPeer: origin,
- }
- q.blockPool[block.Header().Hash()] = block.NumberU64()
- return nil
-}
-
-// Prepare configures the block cache offset to allow accepting inbound blocks.
-func (q *queue) Prepare(offset uint64) {
+// DeliverBodies injects a block body retrieval response into the results queue.
+func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
q.lock.Lock()
defer q.lock.Unlock()
- if q.blockOffset < offset {
- q.blockOffset = offset
+ reconstruct := func(header *types.Header, index int, result *fetchResult) error {
+ if types.DeriveSha(types.Transactions(txLists[index])) != header.TxHash || types.CalcUncleHash(uncleLists[index]) != header.UncleHash {
+ return errInvalidBody
+ }
+ result.Transactions = txLists[index]
+ result.Uncles = uncleLists[index]
+ return nil
+ }
+ return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, q.blockDonePool, bodyReqTimer, len(txLists), reconstruct)
+}
+
+// DeliverReceipts injects a receipt retrieval response into the results queue.
+func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) error {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ reconstruct := func(header *types.Header, index int, result *fetchResult) error {
+ if types.DeriveSha(types.Receipts(receiptList[index])) != header.ReceiptHash {
+ return errInvalidReceipt
+ }
+ result.Receipts = receiptList[index]
+ return nil
+ }
+ return q.deliver(id, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, q.receiptDonePool, receiptReqTimer, len(receiptList), reconstruct)
+}
+
+// deliver injects a data retrieval response into the results queue.
+//
+// Note, this method expects the queue lock to be already held for writing. The
+// reason the lock is not obtained in here is because the parameters already need
+// to access the queue, so they already need a lock anyway.
+func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, pendPool map[string]*fetchRequest,
+ donePool map[common.Hash]struct{}, reqTimer metrics.Timer, results int, reconstruct func(header *types.Header, index int, result *fetchResult) error) error {
+ // Short circuit if the data was never requested
+ request := pendPool[id]
+ if request == nil {
+ return errNoFetchesPending
+ }
+ reqTimer.UpdateSince(request.Time)
+ delete(pendPool, id)
+
+ // If no data items were retrieved, mark them as unavailable for the origin peer
+ if results == 0 {
+ for hash, _ := range request.Headers {
+ request.Peer.ignored.Add(hash)
+ }
+ }
+ // Assemble each of the results with their headers and retrieved data parts
+ var (
+ failure error
+ useful bool
+ )
+ for i, header := range request.Headers {
+ // Short circuit assembly if no more fetch results are found
+ if i >= results {
+ break
+ }
+ // Reconstruct the next result if contents match up
+ index := int(header.Number.Int64() - int64(q.resultOffset))
+ if index >= len(q.resultCache) || index < 0 || q.resultCache[index] == nil {
+ failure = errInvalidChain
+ break
+ }
+ if err := reconstruct(header, i, q.resultCache[index]); err != nil {
+ failure = err
+ break
+ }
+ donePool[header.Hash()] = struct{}{}
+ q.resultCache[index].Pending--
+ useful = true
+
+ // Clean up a successful fetch
+ request.Headers[i] = nil
+ delete(taskPool, header.Hash())
+ }
+ // Return all failed or missing fetches to the queue
+ for _, header := range request.Headers {
+ if header != nil {
+ taskQueue.Push(header, -float32(header.Number.Uint64()))
+ }
+ }
+ // If none of the data was good, it's a stale delivery
+ switch {
+ case failure == nil || failure == errInvalidChain:
+ return failure
+
+ case useful:
+ return fmt.Errorf("partial failure: %v", failure)
+
+ default:
+ return errStaleDelivery
}
}
+
+// DeliverNodeData injects a node state data retrieval response into the queue.
+func (q *queue) DeliverNodeData(id string, data [][]byte, callback func(error, int)) error {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ // Short circuit if the data was never requested
+ request := q.statePendPool[id]
+ if request == nil {
+ return errNoFetchesPending
+ }
+ stateReqTimer.UpdateSince(request.Time)
+ delete(q.statePendPool, id)
+
+ // If no data was retrieved, mark their hashes as unavailable for the origin peer
+ if len(data) == 0 {
+ for hash, _ := range request.Hashes {
+ request.Peer.ignored.Add(hash)
+ }
+ }
+ // Iterate over the downloaded data and verify each of them
+ errs := make([]error, 0)
+ process := []trie.SyncResult{}
+ for _, blob := range data {
+ // Skip any blocks that were not requested
+ hash := common.BytesToHash(crypto.Sha3(blob))
+ if _, ok := request.Hashes[hash]; !ok {
+ errs = append(errs, fmt.Errorf("non-requested state data %x", hash))
+ continue
+ }
+ // Inject the next state trie item into the processing queue
+ process = append(process, trie.SyncResult{hash, blob})
+
+ delete(request.Hashes, hash)
+ delete(q.stateTaskPool, hash)
+ }
+ // Start the asynchronous node state data injection
+ atomic.AddInt32(&q.stateProcessors, 1)
+ go func() {
+ defer atomic.AddInt32(&q.stateProcessors, -1)
+ q.deliverNodeData(process, callback)
+ }()
+ // Return all failed or missing fetches to the queue
+ for hash, index := range request.Hashes {
+ q.stateTaskQueue.Push(hash, float32(index))
+ }
+ // If none of the data items were good, it's a stale delivery
+ switch {
+ case len(errs) == 0:
+ return nil
+
+ case len(errs) == len(request.Hashes):
+ return errStaleDelivery
+
+ default:
+ return fmt.Errorf("multiple failures: %v", errs)
+ }
+}
+
+// deliverNodeData is the asynchronous node data processor that injects a batch
+// of sync results into the state scheduler.
+func (q *queue) deliverNodeData(results []trie.SyncResult, callback func(error, int)) {
+ // Process results one by one to permit task fetches in between
+ for i, result := range results {
+ q.stateSchedLock.Lock()
+
+ if q.stateScheduler == nil {
+ // Syncing aborted since this async delivery started, bail out
+ q.stateSchedLock.Unlock()
+ callback(errNoFetchesPending, i)
+ return
+ }
+ if _, err := q.stateScheduler.Process([]trie.SyncResult{result}); err != nil {
+ // Processing a state result failed, bail out
+ q.stateSchedLock.Unlock()
+ callback(err, i)
+ return
+ }
+ // Item processing succeeded, release the lock (temporarily)
+ q.stateSchedLock.Unlock()
+ }
+ callback(nil, len(results))
+}
+
+// Prepare configures the result cache to allow accepting and caching inbound
+// fetch results.
+func (q *queue) Prepare(offset uint64, mode SyncMode, pivot uint64) {
+ q.lock.Lock()
+ defer q.lock.Unlock()
+
+ if q.resultOffset < offset {
+ q.resultOffset = offset
+ }
+ q.fastSyncPivot = pivot
+ q.mode = mode
+}
diff --git a/eth/downloader/types.go b/eth/downloader/types.go
new file mode 100644
index 000000000..5937be606
--- /dev/null
+++ b/eth/downloader/types.go
@@ -0,0 +1,140 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package downloader
+
+import (
+ "fmt"
+ "math/big"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/core/types"
+)
+
+// headerCheckFn is a callback type for verifying a header's presence in the local chain.
+type headerCheckFn func(common.Hash) bool
+
+// blockCheckFn is a callback type for verifying a block's presence in the local chain.
+type blockCheckFn func(common.Hash) bool
+
+// headerRetrievalFn is a callback type for retrieving a header from the local chain.
+type headerRetrievalFn func(common.Hash) *types.Header
+
+// blockRetrievalFn is a callback type for retrieving a block from the local chain.
+type blockRetrievalFn func(common.Hash) *types.Block
+
+// headHeaderRetrievalFn is a callback type for retrieving the head header from the local chain.
+type headHeaderRetrievalFn func() *types.Header
+
+// headBlockRetrievalFn is a callback type for retrieving the head block from the local chain.
+type headBlockRetrievalFn func() *types.Block
+
+// headFastBlockRetrievalFn is a callback type for retrieving the head fast block from the local chain.
+type headFastBlockRetrievalFn func() *types.Block
+
+// headBlockCommitterFn is a callback for directly committing the head block to a certain entity.
+type headBlockCommitterFn func(common.Hash) error
+
+// tdRetrievalFn is a callback type for retrieving the total difficulty of a local block.
+type tdRetrievalFn func(common.Hash) *big.Int
+
+// headerChainInsertFn is a callback type to insert a batch of headers into the local chain.
+type headerChainInsertFn func([]*types.Header, int) (int, error)
+
+// blockChainInsertFn is a callback type to insert a batch of blocks into the local chain.
+type blockChainInsertFn func(types.Blocks) (int, error)
+
+// receiptChainInsertFn is a callback type to insert a batch of receipts into the local chain.
+type receiptChainInsertFn func(types.Blocks, []types.Receipts) (int, error)
+
+// chainRollbackFn is a callback type to remove a few recently added elements from the local chain.
+type chainRollbackFn func([]common.Hash)
+
+// peerDropFn is a callback type for dropping a peer detected as malicious.
+type peerDropFn func(id string)
+
+// dataPack is a data message returned by a peer for some query.
+type dataPack interface {
+ PeerId() string
+ Items() int
+ Stats() string
+}
+
+// hashPack is a batch of block hashes returned by a peer (eth/61).
+type hashPack struct {
+ peerId string
+ hashes []common.Hash
+}
+
+func (p *hashPack) PeerId() string { return p.peerId }
+func (p *hashPack) Items() int { return len(p.hashes) }
+func (p *hashPack) Stats() string { return fmt.Sprintf("%d", len(p.hashes)) }
+
+// blockPack is a batch of blocks returned by a peer (eth/61).
+type blockPack struct {
+ peerId string
+ blocks []*types.Block
+}
+
+func (p *blockPack) PeerId() string { return p.peerId }
+func (p *blockPack) Items() int { return len(p.blocks) }
+func (p *blockPack) Stats() string { return fmt.Sprintf("%d", len(p.blocks)) }
+
+// headerPack is a batch of block headers returned by a peer.
+type headerPack struct {
+ peerId string
+ headers []*types.Header
+}
+
+func (p *headerPack) PeerId() string { return p.peerId }
+func (p *headerPack) Items() int { return len(p.headers) }
+func (p *headerPack) Stats() string { return fmt.Sprintf("%d", len(p.headers)) }
+
+// bodyPack is a batch of block bodies returned by a peer.
+type bodyPack struct {
+ peerId string
+ transactions [][]*types.Transaction
+ uncles [][]*types.Header
+}
+
+func (p *bodyPack) PeerId() string { return p.peerId }
+func (p *bodyPack) Items() int {
+ if len(p.transactions) <= len(p.uncles) {
+ return len(p.transactions)
+ }
+ return len(p.uncles)
+}
+func (p *bodyPack) Stats() string { return fmt.Sprintf("%d:%d", len(p.transactions), len(p.uncles)) }
+
+// receiptPack is a batch of receipts returned by a peer.
+type receiptPack struct {
+ peerId string
+ receipts [][]*types.Receipt
+}
+
+func (p *receiptPack) PeerId() string { return p.peerId }
+func (p *receiptPack) Items() int { return len(p.receipts) }
+func (p *receiptPack) Stats() string { return fmt.Sprintf("%d", len(p.receipts)) }
+
+// statePack is a batch of states returned by a peer.
+type statePack struct {
+ peerId string
+ states [][]byte
+}
+
+func (p *statePack) PeerId() string { return p.peerId }
+func (p *statePack) Items() int { return len(p.states) }
+func (p *statePack) Stats() string { return fmt.Sprintf("%d", len(p.states)) }
diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go
index b8ec1fc55..d88d91982 100644
--- a/eth/fetcher/fetcher.go
+++ b/eth/fetcher/fetcher.go
@@ -142,9 +142,11 @@ type Fetcher struct {
dropPeer peerDropFn // Drops a peer for misbehaving
// Testing hooks
- fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch
- completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62)
- importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62)
+ announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the announce list
+ queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue
+ fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch
+ completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62)
+ importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62)
}
// New creates a block fetcher to retrieve blocks based on hash announcements.
@@ -324,11 +326,16 @@ func (f *Fetcher) loop() {
height := f.chainHeight()
for !f.queue.Empty() {
op := f.queue.PopItem().(*inject)
-
+ if f.queueChangeHook != nil {
+ f.queueChangeHook(op.block.Hash(), false)
+ }
// If too high up the chain or phase, continue later
number := op.block.NumberU64()
if number > height+1 {
f.queue.Push(op, -float32(op.block.NumberU64()))
+ if f.queueChangeHook != nil {
+ f.queueChangeHook(op.block.Hash(), true)
+ }
break
}
// Otherwise if fresh and still unknown, try and import
@@ -372,6 +379,9 @@ func (f *Fetcher) loop() {
}
f.announces[notification.origin] = count
f.announced[notification.hash] = append(f.announced[notification.hash], notification)
+ if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 {
+ f.announceChangeHook(notification.hash, true)
+ }
if len(f.announced) == 1 {
f.rescheduleFetch(fetchTimer)
}
@@ -714,7 +724,9 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
f.queues[peer] = count
f.queued[hash] = op
f.queue.Push(op, -float32(block.NumberU64()))
-
+ if f.queueChangeHook != nil {
+ f.queueChangeHook(op.block.Hash(), true)
+ }
if glog.V(logger.Debug) {
glog.Infof("Peer %s: queued block #%d [%x…], total %v", peer, block.NumberU64(), hash.Bytes()[:4], f.queue.Size())
}
@@ -781,7 +793,9 @@ func (f *Fetcher) forgetHash(hash common.Hash) {
}
}
delete(f.announced, hash)
-
+ if f.announceChangeHook != nil {
+ f.announceChangeHook(hash, false)
+ }
// Remove any pending fetches and decrement the DOS counters
if announce := f.fetching[hash]; announce != nil {
f.announces[announce.origin]--
diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/fetcher_test.go
index 707d8d758..2404c8cfa 100644
--- a/eth/fetcher/fetcher_test.go
+++ b/eth/fetcher/fetcher_test.go
@@ -45,7 +45,7 @@ var (
// contains a transaction and every 5th an uncle to allow testing correct block
// reassembly.
func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
- blocks := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
+ blocks, _ := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
block.SetCoinbase(common.Address{seed})
// If the block number is multiple of 3, send a bonus transaction to the miner
@@ -145,6 +145,9 @@ func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) {
// dropPeer is an emulator for the peer removal, simply accumulating the various
// peers dropped by the fetcher.
func (f *fetcherTester) dropPeer(peer string) {
+ f.lock.Lock()
+ defer f.lock.Unlock()
+
f.drops[peer] = true
}
@@ -608,8 +611,11 @@ func TestDistantPropagationDiscarding(t *testing.T) {
// Create a tester and simulate a head block being the middle of the above chain
tester := newTester()
+
+ tester.lock.Lock()
tester.hashes = []common.Hash{head}
tester.blocks = map[common.Hash]*types.Block{head: blocks[head]}
+ tester.lock.Unlock()
// Ensure that a block with a lower number than the threshold is discarded
tester.fetcher.Enqueue("lower", blocks[hashes[low]])
@@ -641,8 +647,11 @@ func testDistantAnnouncementDiscarding(t *testing.T, protocol int) {
// Create a tester and simulate a head block being the middle of the above chain
tester := newTester()
+
+ tester.lock.Lock()
tester.hashes = []common.Hash{head}
tester.blocks = map[common.Hash]*types.Block{head: blocks[head]}
+ tester.lock.Unlock()
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
@@ -687,14 +696,22 @@ func testInvalidNumberAnnouncement(t *testing.T, protocol int) {
tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
verifyImportEvent(t, imported, false)
- if !tester.drops["bad"] {
+ tester.lock.RLock()
+ dropped := tester.drops["bad"]
+ tester.lock.RUnlock()
+
+ if !dropped {
t.Fatalf("peer with invalid numbered announcement not dropped")
}
// Make sure a good announcement passes without a drop
tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
verifyImportEvent(t, imported, true)
- if tester.drops["good"] {
+ tester.lock.RLock()
+ dropped = tester.drops["good"]
+ tester.lock.RUnlock()
+
+ if dropped {
t.Fatalf("peer with valid numbered announcement dropped")
}
verifyImportDone(t, imported)
@@ -752,9 +769,15 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) {
// Create a tester with instrumented import hooks
tester := newTester()
- imported := make(chan *types.Block)
+ imported, announces := make(chan *types.Block), int32(0)
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
-
+ tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) {
+ if added {
+ atomic.AddInt32(&announces, 1)
+ } else {
+ atomic.AddInt32(&announces, -1)
+ }
+ }
// Create a valid chain and an infinite junk chain
targetBlocks := hashLimit + 2*maxQueueDist
hashes, blocks := makeChain(targetBlocks, 0, genesis)
@@ -782,8 +805,8 @@ func testHashMemoryExhaustionAttack(t *testing.T, protocol int) {
tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), nil, attackerHeaderFetcher, attackerBodyFetcher)
}
}
- if len(tester.fetcher.announced) != hashLimit+maxQueueDist {
- t.Fatalf("queued announce count mismatch: have %d, want %d", len(tester.fetcher.announced), hashLimit+maxQueueDist)
+ if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist {
+ t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist)
}
// Wait for fetches to complete
verifyImportCount(t, imported, maxQueueDist)
@@ -807,9 +830,15 @@ func TestBlockMemoryExhaustionAttack(t *testing.T) {
// Create a tester with instrumented import hooks
tester := newTester()
- imported := make(chan *types.Block)
+ imported, enqueued := make(chan *types.Block), int32(0)
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
-
+ tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) {
+ if added {
+ atomic.AddInt32(&enqueued, 1)
+ } else {
+ atomic.AddInt32(&enqueued, -1)
+ }
+ }
// Create a valid chain and a batch of dangling (but in range) blocks
targetBlocks := hashLimit + 2*maxQueueDist
hashes, blocks := makeChain(targetBlocks, 0, genesis)
@@ -825,7 +854,7 @@ func TestBlockMemoryExhaustionAttack(t *testing.T) {
tester.fetcher.Enqueue("attacker", block)
}
time.Sleep(200 * time.Millisecond)
- if queued := tester.fetcher.queue.Size(); queued != blockLimit {
+ if queued := atomic.LoadInt32(&enqueued); queued != blockLimit {
t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit)
}
// Queue up a batch of valid blocks, and check that a new peer is allowed to do so
@@ -833,7 +862,7 @@ func TestBlockMemoryExhaustionAttack(t *testing.T) {
tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]])
}
time.Sleep(100 * time.Millisecond)
- if queued := tester.fetcher.queue.Size(); queued != blockLimit+maxQueueDist-1 {
+ if queued := atomic.LoadInt32(&enqueued); queued != blockLimit+maxQueueDist-1 {
t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1)
}
// Insert the missing piece (and sanity check the import)
diff --git a/eth/filters/filter.go b/eth/filters/filter.go
index 2e81ea177..ff192cdf6 100644
--- a/eth/filters/filter.go
+++ b/eth/filters/filter.go
@@ -138,7 +138,7 @@ func (self *Filter) getLogs(start, end uint64) (logs vm.Logs) {
unfiltered vm.Logs
)
for _, receipt := range receipts {
- unfiltered = append(unfiltered, receipt.Logs()...)
+ unfiltered = append(unfiltered, receipt.Logs...)
}
logs = append(logs, self.FilterLogs(unfiltered)...)
}
diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go
index 9e7538fac..a5418e2e7 100644
--- a/eth/filters/filter_test.go
+++ b/eth/filters/filter_test.go
@@ -16,9 +16,9 @@ import (
func makeReceipt(addr common.Address) *types.Receipt {
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{Address: addr},
- })
+ }
receipt.Bloom = types.CreateBloom(types.Receipts{receipt})
return receipt
}
@@ -41,7 +41,7 @@ func BenchmarkMipmaps(b *testing.B) {
defer db.Close()
genesis := core.WriteGenesisBlockForTesting(db, core.GenesisAccount{addr1, big.NewInt(1000000)})
- chain := core.GenerateChain(genesis, db, 100010, func(i int, gen *core.BlockGen) {
+ chain, receipts := core.GenerateChain(genesis, db, 100010, func(i int, gen *core.BlockGen) {
var receipts types.Receipts
switch i {
case 2403:
@@ -70,7 +70,7 @@ func BenchmarkMipmaps(b *testing.B) {
}
core.WriteMipmapBloom(db, uint64(i+1), receipts)
})
- for _, block := range chain {
+ for i, block := range chain {
core.WriteBlock(db, block)
if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
b.Fatalf("failed to insert block number: %v", err)
@@ -78,11 +78,10 @@ func BenchmarkMipmaps(b *testing.B) {
if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil {
b.Fatalf("failed to insert block number: %v", err)
}
- if err := core.PutBlockReceipts(db, block, block.Receipts()); err != nil {
+ if err := core.PutBlockReceipts(db, block.Hash(), receipts[i]); err != nil {
b.Fatal("error writing block receipts:", err)
}
}
-
b.ResetTimer()
filter := New(db)
@@ -118,47 +117,47 @@ func TestFilters(t *testing.T) {
defer db.Close()
genesis := core.WriteGenesisBlockForTesting(db, core.GenesisAccount{addr, big.NewInt(1000000)})
- chain := core.GenerateChain(genesis, db, 1000, func(i int, gen *core.BlockGen) {
+ chain, receipts := core.GenerateChain(genesis, db, 1000, func(i int, gen *core.BlockGen) {
var receipts types.Receipts
switch i {
case 1:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{
Address: addr,
Topics: []common.Hash{hash1},
},
- })
+ }
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 2:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{
Address: addr,
Topics: []common.Hash{hash2},
},
- })
+ }
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 998:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{
Address: addr,
Topics: []common.Hash{hash3},
},
- })
+ }
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
case 999:
receipt := types.NewReceipt(nil, new(big.Int))
- receipt.SetLogs(vm.Logs{
+ receipt.Logs = vm.Logs{
&vm.Log{
Address: addr,
Topics: []common.Hash{hash4},
},
- })
+ }
gen.AddUncheckedReceipt(receipt)
receipts = types.Receipts{receipt}
}
@@ -173,7 +172,7 @@ func TestFilters(t *testing.T) {
// by one
core.WriteMipmapBloom(db, uint64(i+1), receipts)
})
- for _, block := range chain {
+ for i, block := range chain {
core.WriteBlock(db, block)
if err := core.WriteCanonicalHash(db, block.Hash(), block.NumberU64()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
@@ -181,7 +180,7 @@ func TestFilters(t *testing.T) {
if err := core.WriteHeadBlockHash(db, block.Hash()); err != nil {
t.Fatalf("failed to insert block number: %v", err)
}
- if err := core.PutBlockReceipts(db, block, block.Receipts()); err != nil {
+ if err := core.PutBlockReceipts(db, block.Hash(), receipts[i]); err != nil {
t.Fatal("error writing block receipts:", err)
}
}
diff --git a/eth/handler.go b/eth/handler.go
index 3fc909672..7dc7de80e 100644
--- a/eth/handler.go
+++ b/eth/handler.go
@@ -17,6 +17,7 @@
package eth
import (
+ "errors"
"fmt"
"math"
"math/big"
@@ -42,6 +43,10 @@ const (
estHeaderRlpSize = 500 // Approximate size of an RLP encoded block header
)
+// errIncompatibleConfig is returned if the requested protocols and configs are
+// not compatible (low protocol version restrictions and high requirements).
+var errIncompatibleConfig = errors.New("incompatible configuration")
+
func errResp(code errCode, format string, v ...interface{}) error {
return fmt.Errorf("%v - %v", code, fmt.Sprintf(format, v...))
}
@@ -49,17 +54,8 @@ func errResp(code errCode, format string, v ...interface{}) error {
type hashFetcherFn func(common.Hash) error
type blockFetcherFn func([]common.Hash) error
-// extProt is an interface which is passed around so we can expose GetHashes and GetBlock without exposing it to the rest of the protocol
-// extProt is passed around to peers which require to GetHashes and GetBlocks
-type extProt struct {
- getHashes hashFetcherFn
- getBlocks blockFetcherFn
-}
-
-func (ep extProt) GetHashes(hash common.Hash) error { return ep.getHashes(hash) }
-func (ep extProt) GetBlock(hashes []common.Hash) error { return ep.getBlocks(hashes) }
-
type ProtocolManager struct {
+ fastSync bool
txpool txPool
blockchain *core.BlockChain
chaindb ethdb.Database
@@ -87,9 +83,15 @@ type ProtocolManager struct {
// NewProtocolManager returns a new ethereum sub protocol manager. The Ethereum sub protocol manages peers capable
// with the ethereum network.
-func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, blockchain *core.BlockChain, chaindb ethdb.Database) *ProtocolManager {
+func NewProtocolManager(fastSync bool, networkId int, mux *event.TypeMux, txpool txPool, pow pow.PoW, blockchain *core.BlockChain, chaindb ethdb.Database) (*ProtocolManager, error) {
+ // Figure out whether to allow fast sync or not
+ if fastSync && blockchain.CurrentBlock().NumberU64() > 0 {
+ glog.V(logger.Info).Infof("blockchain not empty, fast sync disabled")
+ fastSync = false
+ }
// Create the protocol manager with the base fields
manager := &ProtocolManager{
+ fastSync: fastSync,
eventMux: mux,
txpool: txpool,
blockchain: blockchain,
@@ -100,11 +102,15 @@ func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow po
quitSync: make(chan struct{}),
}
// Initiate a sub-protocol for every implemented version we can handle
- manager.SubProtocols = make([]p2p.Protocol, len(ProtocolVersions))
- for i := 0; i < len(manager.SubProtocols); i++ {
- version := ProtocolVersions[i]
-
- manager.SubProtocols[i] = p2p.Protocol{
+ manager.SubProtocols = make([]p2p.Protocol, 0, len(ProtocolVersions))
+ for i, version := range ProtocolVersions {
+ // Skip protocol version if incompatible with the mode of operation
+ if fastSync && version < eth63 {
+ continue
+ }
+ // Compatible; initialise the sub-protocol
+ version := version // Closure for the run
+ manager.SubProtocols = append(manager.SubProtocols, p2p.Protocol{
Name: "eth",
Version: version,
Length: ProtocolLengths[i],
@@ -113,20 +119,25 @@ func NewProtocolManager(networkId int, mux *event.TypeMux, txpool txPool, pow po
manager.newPeerCh <- peer
return manager.handle(peer)
},
- }
+ })
+ }
+ if len(manager.SubProtocols) == 0 {
+ return nil, errIncompatibleConfig
}
// Construct the different synchronisation mechanisms
- manager.downloader = downloader.New(manager.eventMux, manager.blockchain.HasBlock, manager.blockchain.GetBlock, manager.blockchain.CurrentBlock, manager.blockchain.GetTd, manager.blockchain.InsertChain, manager.removePeer)
+ manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlock, blockchain.GetHeader, blockchain.GetBlock,
+ blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, blockchain.GetTd,
+ blockchain.InsertHeaderChain, blockchain.InsertChain, blockchain.InsertReceiptChain, blockchain.Rollback, manager.removePeer)
validator := func(block *types.Block, parent *types.Block) error {
return core.ValidateHeader(pow, block.Header(), parent.Header(), true, false)
}
heighter := func() uint64 {
- return manager.blockchain.CurrentBlock().NumberU64()
+ return blockchain.CurrentBlock().NumberU64()
}
- manager.fetcher = fetcher.New(manager.blockchain.GetBlock, validator, manager.BroadcastBlock, heighter, manager.blockchain.InsertChain, manager.removePeer)
+ manager.fetcher = fetcher.New(blockchain.GetBlock, validator, manager.BroadcastBlock, heighter, blockchain.InsertChain, manager.removePeer)
- return manager
+ return manager, nil
}
func (pm *ProtocolManager) removePeer(id string) {
@@ -205,8 +216,8 @@ func (pm *ProtocolManager) handle(p *peer) error {
// Register the peer in the downloader. If the downloader considers it banned, we disconnect
if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(),
- p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks,
- p.RequestHeadersByHash, p.RequestHeadersByNumber, p.RequestBodies); err != nil {
+ p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks, p.RequestHeadersByHash,
+ p.RequestHeadersByNumber, p.RequestBodies, p.RequestReceipts, p.RequestNodeData); err != nil {
return err
}
// Propagate existing transactions. new transactions appearing
@@ -292,7 +303,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
break
}
// Deliver them all to the downloader for queuing
- err := pm.downloader.DeliverHashes61(p.id, hashes)
+ err := pm.downloader.DeliverHashes(p.id, hashes)
if err != nil {
glog.V(logger.Debug).Infoln(err)
}
@@ -338,7 +349,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
// Filter out any explicitly requested blocks, deliver the rest to the downloader
if blocks := pm.fetcher.FilterBlocks(blocks); len(blocks) > 0 {
- pm.downloader.DeliverBlocks61(p.id, blocks)
+ pm.downloader.DeliverBlocks(p.id, blocks)
}
// Block header query, collect the requested headers and reply
@@ -424,28 +435,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
}
- case p.version >= eth62 && msg.Code == BlockBodiesMsg:
- // A batch of block bodies arrived to one of our previous requests
- var request blockBodiesData
- if err := msg.Decode(&request); err != nil {
- return errResp(ErrDecode, "msg %v: %v", msg, err)
- }
- // Deliver them all to the downloader for queuing
- trasactions := make([][]*types.Transaction, len(request))
- uncles := make([][]*types.Header, len(request))
-
- for i, body := range request {
- trasactions[i] = body.Transactions
- uncles[i] = body.Uncles
- }
- // Filter out any explicitly requested bodies, deliver the rest to the downloader
- if trasactions, uncles := pm.fetcher.FilterBodies(trasactions, uncles, time.Now()); len(trasactions) > 0 || len(uncles) > 0 {
- err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
- if err != nil {
- glog.V(logger.Debug).Infoln(err)
- }
- }
-
case p.version >= eth62 && msg.Code == GetBlockBodiesMsg:
// Decode the retrieval message
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
@@ -473,6 +462,28 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
return p.SendBlockBodiesRLP(bodies)
+ case p.version >= eth62 && msg.Code == BlockBodiesMsg:
+ // A batch of block bodies arrived to one of our previous requests
+ var request blockBodiesData
+ if err := msg.Decode(&request); err != nil {
+ return errResp(ErrDecode, "msg %v: %v", msg, err)
+ }
+ // Deliver them all to the downloader for queuing
+ trasactions := make([][]*types.Transaction, len(request))
+ uncles := make([][]*types.Header, len(request))
+
+ for i, body := range request {
+ trasactions[i] = body.Transactions
+ uncles[i] = body.Uncles
+ }
+ // Filter out any explicitly requested bodies, deliver the rest to the downloader
+ if trasactions, uncles := pm.fetcher.FilterBodies(trasactions, uncles, time.Now()); len(trasactions) > 0 || len(uncles) > 0 {
+ err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
+ if err != nil {
+ glog.V(logger.Debug).Infoln(err)
+ }
+ }
+
case p.version >= eth63 && msg.Code == GetNodeDataMsg:
// Decode the retrieval message
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
@@ -500,6 +511,17 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
return p.SendNodeData(data)
+ case p.version >= eth63 && msg.Code == NodeDataMsg:
+ // A batch of node state data arrived to one of our previous requests
+ var data [][]byte
+ if err := msg.Decode(&data); err != nil {
+ return errResp(ErrDecode, "msg %v: %v", msg, err)
+ }
+ // Deliver all to the downloader
+ if err := pm.downloader.DeliverNodeData(p.id, data); err != nil {
+ glog.V(logger.Debug).Infof("failed to deliver node state data: %v", err)
+ }
+
case p.version >= eth63 && msg.Code == GetReceiptsMsg:
// Decode the retrieval message
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
@@ -510,22 +532,42 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
var (
hash common.Hash
bytes int
- receipts []*types.Receipt
+ receipts []rlp.RawValue
)
- for bytes < softResponseLimit && len(receipts) < downloader.MaxReceiptsFetch {
- // Retrieve the hash of the next transaction receipt
+ for bytes < softResponseLimit && len(receipts) < downloader.MaxReceiptFetch {
+ // Retrieve the hash of the next block
if err := msgStream.Decode(&hash); err == rlp.EOL {
break
} else if err != nil {
return errResp(ErrDecode, "msg %v: %v", msg, err)
}
- // Retrieve the requested receipt, stopping if enough was found
- if receipt := core.GetReceipt(pm.chaindb, hash); receipt != nil {
- receipts = append(receipts, receipt)
- bytes += len(receipt.RlpEncode())
+ // Retrieve the requested block's receipts, skipping if unknown to us
+ results := core.GetBlockReceipts(pm.chaindb, hash)
+ if results == nil {
+ if header := pm.blockchain.GetHeader(hash); header == nil || header.ReceiptHash != types.EmptyRootHash {
+ continue
+ }
+ }
+ // If known, encode and queue for response packet
+ if encoded, err := rlp.EncodeToBytes(results); err != nil {
+ glog.V(logger.Error).Infof("failed to encode receipt: %v", err)
+ } else {
+ receipts = append(receipts, encoded)
+ bytes += len(encoded)
}
}
- return p.SendReceipts(receipts)
+ return p.SendReceiptsRLP(receipts)
+
+ case p.version >= eth63 && msg.Code == ReceiptsMsg:
+ // A batch of receipts arrived to one of our previous requests
+ var receipts [][]*types.Receipt
+ if err := msg.Decode(&receipts); err != nil {
+ return errResp(ErrDecode, "msg %v: %v", msg, err)
+ }
+ // Deliver all to the downloader
+ if err := pm.downloader.DeliverReceipts(p.id, receipts); err != nil {
+ glog.V(logger.Debug).Infof("failed to deliver receipts: %v", err)
+ }
case msg.Code == NewBlockHashesMsg:
// Retrieve and deseralize the remote new block hashes notification
@@ -585,15 +627,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
}
request.Block.ReceivedAt = msg.ReceivedAt
- // Mark the block's arrival for whatever reason
- _, chainHead, _ := pm.blockchain.Status()
- jsonlogger.LogJson(&logger.EthChainReceivedNewBlock{
- BlockHash: request.Block.Hash().Hex(),
- BlockNumber: request.Block.Number(),
- ChainHeadHash: chainHead.Hex(),
- BlockPrevHash: request.Block.ParentHash().Hex(),
- RemoteId: p.ID().String(),
- })
// Mark the peer as owning the block and schedule it for import
p.MarkBlock(request.Block.Hash())
p.SetHead(request.Block.Hash())
@@ -603,7 +636,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
// Update the peers total difficulty if needed, schedule a download if gapped
if request.TD.Cmp(p.Td()) > 0 {
p.SetTd(request.TD)
- if request.TD.Cmp(new(big.Int).Add(pm.blockchain.Td(), request.Block.Difficulty())) > 0 {
+ td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash())
+ if request.TD.Cmp(new(big.Int).Add(td, request.Block.Difficulty())) > 0 {
go pm.synchronise(p)
}
}
@@ -620,12 +654,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
return errResp(ErrDecode, "transaction %d is nil", i)
}
p.MarkTransaction(tx.Hash())
-
- // Log it's arrival for later analysis
- jsonlogger.LogJson(&logger.EthTxReceived{
- TxHash: tx.Hash().Hex(),
- RemoteId: p.ID().String(),
- })
}
pm.txpool.AddTransactions(txs)
diff --git a/eth/handler_test.go b/eth/handler_test.go
index dde2ecbd5..ab2ce54b1 100644
--- a/eth/handler_test.go
+++ b/eth/handler_test.go
@@ -17,12 +17,41 @@ import (
"github.com/ethereum/go-ethereum/params"
)
+// Tests that protocol versions and modes of operations are matched up properly.
+func TestProtocolCompatibility(t *testing.T) {
+ // Define the compatibility chart
+ tests := []struct {
+ version uint
+ fastSync bool
+ compatible bool
+ }{
+ {61, false, true}, {62, false, true}, {63, false, true},
+ {61, true, false}, {62, true, false}, {63, true, true},
+ }
+ // Make sure anything we screw up is restored
+ backup := ProtocolVersions
+ defer func() { ProtocolVersions = backup }()
+
+ // Try all available compatibility configs and check for errors
+ for i, tt := range tests {
+ ProtocolVersions = []uint{tt.version}
+
+ pm, err := newTestProtocolManager(tt.fastSync, 0, nil, nil)
+ if pm != nil {
+ defer pm.Stop()
+ }
+ if (err == nil && !tt.compatible) || (err != nil && tt.compatible) {
+ t.Errorf("test %d: compatibility mismatch: have error %v, want compatibility %v", i, err, tt.compatible)
+ }
+ }
+}
+
// Tests that hashes can be retrieved from a remote chain by hashes in reverse
// order.
func TestGetBlockHashes61(t *testing.T) { testGetBlockHashes(t, 61) }
func testGetBlockHashes(t *testing.T, protocol int) {
- pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@@ -65,7 +94,7 @@ func testGetBlockHashes(t *testing.T, protocol int) {
func TestGetBlockHashesFromNumber61(t *testing.T) { testGetBlockHashesFromNumber(t, 61) }
func testGetBlockHashesFromNumber(t *testing.T, protocol int) {
- pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@@ -105,7 +134,7 @@ func testGetBlockHashesFromNumber(t *testing.T, protocol int) {
func TestGetBlocks61(t *testing.T) { testGetBlocks(t, 61) }
func testGetBlocks(t *testing.T, protocol int) {
- pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@@ -174,10 +203,9 @@ func testGetBlocks(t *testing.T, protocol int) {
// Tests that block headers can be retrieved from a remote chain based on user queries.
func TestGetBlockHeaders62(t *testing.T) { testGetBlockHeaders(t, 62) }
func TestGetBlockHeaders63(t *testing.T) { testGetBlockHeaders(t, 63) }
-func TestGetBlockHeaders64(t *testing.T) { testGetBlockHeaders(t, 64) }
func testGetBlockHeaders(t *testing.T, protocol int) {
- pm := newTestProtocolManager(downloader.MaxHashFetch+15, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, downloader.MaxHashFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@@ -300,10 +328,9 @@ func testGetBlockHeaders(t *testing.T, protocol int) {
// Tests that block contents can be retrieved from a remote chain based on their hashes.
func TestGetBlockBodies62(t *testing.T) { testGetBlockBodies(t, 62) }
func TestGetBlockBodies63(t *testing.T) { testGetBlockBodies(t, 63) }
-func TestGetBlockBodies64(t *testing.T) { testGetBlockBodies(t, 64) }
func testGetBlockBodies(t *testing.T, protocol int) {
- pm := newTestProtocolManager(downloader.MaxBlockFetch+15, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, downloader.MaxBlockFetch+15, nil, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
@@ -372,7 +399,6 @@ func testGetBlockBodies(t *testing.T, protocol int) {
// Tests that the node state database can be retrieved based on hashes.
func TestGetNodeData63(t *testing.T) { testGetNodeData(t, 63) }
-func TestGetNodeData64(t *testing.T) { testGetNodeData(t, 64) }
func testGetNodeData(t *testing.T, protocol int) {
// Define three accounts to simulate transactions with
@@ -410,14 +436,16 @@ func testGetNodeData(t *testing.T, protocol int) {
}
}
// Assemble the test environment
- pm := newTestProtocolManager(4, generator, nil)
+ pm := newTestProtocolManagerMust(t, false, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
// Fetch for now the entire chain db
hashes := []common.Hash{}
for _, key := range pm.chaindb.(*ethdb.MemDatabase).Keys() {
- hashes = append(hashes, common.BytesToHash(key))
+ if len(key) == len(common.Hash{}) {
+ hashes = append(hashes, common.BytesToHash(key))
+ }
}
p2p.Send(peer.app, 0x0d, hashes)
msg, err := peer.app.ReadMsg()
@@ -462,7 +490,6 @@ func testGetNodeData(t *testing.T, protocol int) {
// Tests that the transaction receipts can be retrieved based on hashes.
func TestGetReceipt63(t *testing.T) { testGetReceipt(t, 63) }
-func TestGetReceipt64(t *testing.T) { testGetReceipt(t, 64) }
func testGetReceipt(t *testing.T, protocol int) {
// Define three accounts to simulate transactions with
@@ -500,20 +527,17 @@ func testGetReceipt(t *testing.T, protocol int) {
}
}
// Assemble the test environment
- pm := newTestProtocolManager(4, generator, nil)
+ pm := newTestProtocolManagerMust(t, false, 4, generator, nil)
peer, _ := newTestPeer("peer", protocol, pm, true)
defer peer.close()
// Collect the hashes to request, and the response to expect
- hashes := []common.Hash{}
+ hashes, receipts := []common.Hash{}, []types.Receipts{}
for i := uint64(0); i <= pm.blockchain.CurrentBlock().NumberU64(); i++ {
- for _, tx := range pm.blockchain.GetBlockByNumber(i).Transactions() {
- hashes = append(hashes, tx.Hash())
- }
- }
- receipts := make([]*types.Receipt, len(hashes))
- for i, hash := range hashes {
- receipts[i] = core.GetReceipt(pm.chaindb, hash)
+ block := pm.blockchain.GetBlockByNumber(i)
+
+ hashes = append(hashes, block.Hash())
+ receipts = append(receipts, core.GetBlockReceipts(pm.chaindb, block.Hash()))
}
// Send the hash request and verify the response
p2p.Send(peer.app, 0x0f, hashes)
diff --git a/eth/helper_test.go b/eth/helper_test.go
index 9314884ef..16907be8b 100644
--- a/eth/helper_test.go
+++ b/eth/helper_test.go
@@ -28,7 +28,7 @@ var (
// newTestProtocolManager creates a new protocol manager for testing purposes,
// with the given number of blocks already known, and potential notification
// channels for different events.
-func newTestProtocolManager(blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) *ProtocolManager {
+func newTestProtocolManager(fastSync bool, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) (*ProtocolManager, error) {
var (
evmux = new(event.TypeMux)
pow = new(core.FakePow)
@@ -38,12 +38,27 @@ func newTestProtocolManager(blocks int, generator func(int, *core.BlockGen), new
blockproc = core.NewBlockProcessor(db, pow, blockchain, evmux)
)
blockchain.SetProcessor(blockproc)
- chain := core.GenerateChain(genesis, db, blocks, generator)
+ chain, _ := core.GenerateChain(genesis, db, blocks, generator)
if _, err := blockchain.InsertChain(chain); err != nil {
panic(err)
}
- pm := NewProtocolManager(NetworkId, evmux, &testTxPool{added: newtx}, pow, blockchain, db)
+ pm, err := NewProtocolManager(fastSync, NetworkId, evmux, &testTxPool{added: newtx}, pow, blockchain, db)
+ if err != nil {
+ return nil, err
+ }
pm.Start()
+ return pm, nil
+}
+
+// newTestProtocolManagerMust creates a new protocol manager for testing purposes,
+// with the given number of blocks already known, and potential notification
+// channels for different events. In case of an error, the constructor force-
+// fails the test.
+func newTestProtocolManagerMust(t *testing.T, fastSync bool, blocks int, generator func(int, *core.BlockGen), newtx chan<- []*types.Transaction) *ProtocolManager {
+ pm, err := newTestProtocolManager(fastSync, blocks, generator, newtx)
+ if err != nil {
+ t.Fatalf("Failed to create protocol manager: %v", err)
+ }
return pm
}
diff --git a/eth/metrics.go b/eth/metrics.go
index cfab3bcb3..8231a06ff 100644
--- a/eth/metrics.go
+++ b/eth/metrics.go
@@ -101,7 +101,7 @@ func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) {
packets, traffic = reqBlockInPacketsMeter, reqBlockInTrafficMeter
case rw.version >= eth62 && msg.Code == BlockHeadersMsg:
- packets, traffic = reqBlockInPacketsMeter, reqBlockInTrafficMeter
+ packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter
case rw.version >= eth62 && msg.Code == BlockBodiesMsg:
packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter
diff --git a/eth/peer.go b/eth/peer.go
index 603b49b88..68ce903a6 100644
--- a/eth/peer.go
+++ b/eth/peer.go
@@ -191,15 +191,15 @@ func (p *peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error {
return p2p.Send(p.rw, BlockBodiesMsg, bodies)
}
-// SendNodeData sends a batch of arbitrary internal data, corresponding to the
+// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the
// hashes requested.
func (p *peer) SendNodeData(data [][]byte) error {
return p2p.Send(p.rw, NodeDataMsg, data)
}
-// SendReceipts sends a batch of transaction receipts, corresponding to the ones
-// requested.
-func (p *peer) SendReceipts(receipts []*types.Receipt) error {
+// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the
+// ones requested from an already RLP encoded format.
+func (p *peer) SendReceiptsRLP(receipts []rlp.RawValue) error {
return p2p.Send(p.rw, ReceiptsMsg, receipts)
}
diff --git a/eth/protocol.go b/eth/protocol.go
index 49f096a3b..410347ed3 100644
--- a/eth/protocol.go
+++ b/eth/protocol.go
@@ -31,14 +31,13 @@ const (
eth61 = 61
eth62 = 62
eth63 = 63
- eth64 = 64
)
// Supported versions of the eth protocol (first is primary).
-var ProtocolVersions = []uint{eth64, eth63, eth62, eth61}
+var ProtocolVersions = []uint{eth63, eth62, eth61}
// Number of implemented message corresponding to different protocol versions.
-var ProtocolLengths = []uint64{15, 12, 8, 9}
+var ProtocolLengths = []uint64{17, 8, 9}
const (
NetworkId = 1
@@ -73,11 +72,6 @@ const (
NodeDataMsg = 0x0e
GetReceiptsMsg = 0x0f
ReceiptsMsg = 0x10
-
- // Protocol messages belonging to eth/64
- GetAcctProofMsg = 0x11
- GetStorageDataProof = 0x12
- Proof = 0x13
)
type errCode int
diff --git a/eth/protocol_test.go b/eth/protocol_test.go
index 523e6c1eb..372c7e203 100644
--- a/eth/protocol_test.go
+++ b/eth/protocol_test.go
@@ -41,10 +41,9 @@ var testAccount = crypto.NewKey(rand.Reader)
func TestStatusMsgErrors61(t *testing.T) { testStatusMsgErrors(t, 61) }
func TestStatusMsgErrors62(t *testing.T) { testStatusMsgErrors(t, 62) }
func TestStatusMsgErrors63(t *testing.T) { testStatusMsgErrors(t, 63) }
-func TestStatusMsgErrors64(t *testing.T) { testStatusMsgErrors(t, 64) }
func testStatusMsgErrors(t *testing.T, protocol int) {
- pm := newTestProtocolManager(0, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, 0, nil, nil)
td, currentBlock, genesis := pm.blockchain.Status()
defer pm.Stop()
@@ -95,11 +94,10 @@ func testStatusMsgErrors(t *testing.T, protocol int) {
func TestRecvTransactions61(t *testing.T) { testRecvTransactions(t, 61) }
func TestRecvTransactions62(t *testing.T) { testRecvTransactions(t, 62) }
func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) }
-func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) }
func testRecvTransactions(t *testing.T, protocol int) {
txAdded := make(chan []*types.Transaction)
- pm := newTestProtocolManager(0, nil, txAdded)
+ pm := newTestProtocolManagerMust(t, false, 0, nil, txAdded)
p, _ := newTestPeer("peer", protocol, pm, true)
defer pm.Stop()
defer p.close()
@@ -124,10 +122,9 @@ func testRecvTransactions(t *testing.T, protocol int) {
func TestSendTransactions61(t *testing.T) { testSendTransactions(t, 61) }
func TestSendTransactions62(t *testing.T) { testSendTransactions(t, 62) }
func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) }
-func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) }
func testSendTransactions(t *testing.T, protocol int) {
- pm := newTestProtocolManager(0, nil, nil)
+ pm := newTestProtocolManagerMust(t, false, 0, nil, nil)
defer pm.Stop()
// Fill the pool with big transactions.
diff --git a/eth/sync.go b/eth/sync.go
index 5a2031c68..b69a24556 100644
--- a/eth/sync.go
+++ b/eth/sync.go
@@ -22,6 +22,7 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/eth/downloader"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
"github.com/ethereum/go-ethereum/p2p/discover"
@@ -160,9 +161,25 @@ func (pm *ProtocolManager) synchronise(peer *peer) {
return
}
// Make sure the peer's TD is higher than our own. If not drop.
- if peer.Td().Cmp(pm.blockchain.Td()) <= 0 {
+ td := pm.blockchain.GetTd(pm.blockchain.CurrentBlock().Hash())
+ if peer.Td().Cmp(td) <= 0 {
return
}
// Otherwise try to sync with the downloader
- pm.downloader.Synchronise(peer.id, peer.Head(), peer.Td())
+ mode := downloader.FullSync
+ if pm.fastSync {
+ mode = downloader.FastSync
+ }
+ pm.downloader.Synchronise(peer.id, peer.Head(), peer.Td(), mode)
+
+ // If fast sync was enabled, and we synced up, disable it
+ if pm.fastSync {
+ for pm.downloader.Synchronising() {
+ time.Sleep(100 * time.Millisecond)
+ }
+ if pm.blockchain.CurrentBlock().NumberU64() > 0 {
+ glog.V(logger.Info).Infof("fast sync complete, auto disabling")
+ pm.fastSync = false
+ }
+ }
}
diff --git a/eth/sync_test.go b/eth/sync_test.go
new file mode 100644
index 000000000..f3a6718ab
--- /dev/null
+++ b/eth/sync_test.go
@@ -0,0 +1,53 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package eth
+
+import (
+ "testing"
+ "time"
+
+ "github.com/ethereum/go-ethereum/p2p"
+ "github.com/ethereum/go-ethereum/p2p/discover"
+)
+
+// Tests that fast sync gets disabled as soon as a real block is successfully
+// imported into the blockchain.
+func TestFastSyncDisabling(t *testing.T) {
+ // Create a pristine protocol manager, check that fast sync is left enabled
+ pmEmpty := newTestProtocolManagerMust(t, true, 0, nil, nil)
+ if !pmEmpty.fastSync {
+ t.Fatalf("fast sync disabled on pristine blockchain")
+ }
+ // Create a full protocol manager, check that fast sync gets disabled
+ pmFull := newTestProtocolManagerMust(t, true, 1024, nil, nil)
+ if pmFull.fastSync {
+ t.Fatalf("fast sync not disabled on non-empty blockchain")
+ }
+ // Sync up the two peers
+ io1, io2 := p2p.MsgPipe()
+
+ go pmFull.handle(pmFull.newPeer(63, NetworkId, p2p.NewPeer(discover.NodeID{}, "empty", nil), io2))
+ go pmEmpty.handle(pmEmpty.newPeer(63, NetworkId, p2p.NewPeer(discover.NodeID{}, "full", nil), io1))
+
+ time.Sleep(250 * time.Millisecond)
+ pmEmpty.synchronise(pmEmpty.peers.BestPeer())
+
+ // Check that fast sync was disabled
+ if pmEmpty.fastSync {
+ t.Fatalf("fast sync not disabled after successful synchronisation")
+ }
+}
diff --git a/ethdb/memory_database.go b/ethdb/memory_database.go
index 81911f23f..01273b9db 100644
--- a/ethdb/memory_database.go
+++ b/ethdb/memory_database.go
@@ -17,7 +17,9 @@
package ethdb
import (
+ "errors"
"fmt"
+ "sync"
"github.com/ethereum/go-ethereum/common"
)
@@ -26,29 +28,45 @@ import (
* This is a test memory database. Do not use for any production it does not get persisted
*/
type MemDatabase struct {
- db map[string][]byte
+ db map[string][]byte
+ lock sync.RWMutex
}
func NewMemDatabase() (*MemDatabase, error) {
- db := &MemDatabase{db: make(map[string][]byte)}
-
- return db, nil
+ return &MemDatabase{
+ db: make(map[string][]byte),
+ }, nil
}
func (db *MemDatabase) Put(key []byte, value []byte) error {
+ db.lock.Lock()
+ defer db.lock.Unlock()
+
db.db[string(key)] = common.CopyBytes(value)
return nil
}
func (db *MemDatabase) Set(key []byte, value []byte) {
+ db.lock.Lock()
+ defer db.lock.Unlock()
+
db.Put(key, value)
}
func (db *MemDatabase) Get(key []byte) ([]byte, error) {
- return db.db[string(key)], nil
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
+ if entry, ok := db.db[string(key)]; ok {
+ return entry, nil
+ }
+ return nil, errors.New("not found")
}
func (db *MemDatabase) Keys() [][]byte {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
keys := [][]byte{}
for key, _ := range db.db {
keys = append(keys, []byte(key))
@@ -65,12 +83,17 @@ func (db *MemDatabase) GetKeys() []*common.Key {
*/
func (db *MemDatabase) Delete(key []byte) error {
- delete(db.db, string(key))
+ db.lock.Lock()
+ defer db.lock.Unlock()
+ delete(db.db, string(key))
return nil
}
func (db *MemDatabase) Print() {
+ db.lock.RLock()
+ defer db.lock.RUnlock()
+
for key, val := range db.db {
fmt.Printf("%x(%d): ", key, len(key))
node := common.NewValueFromBytes(val)
@@ -83,11 +106,9 @@ func (db *MemDatabase) Close() {
func (db *MemDatabase) LastKnownTD() []byte {
data, _ := db.Get([]byte("LastKnownTotalDifficulty"))
-
if len(data) == 0 || data == nil {
data = []byte{0x0}
}
-
return data
}
@@ -100,16 +121,26 @@ type kv struct{ k, v []byte }
type memBatch struct {
db *MemDatabase
writes []kv
+ lock sync.RWMutex
}
-func (w *memBatch) Put(key, value []byte) error {
- w.writes = append(w.writes, kv{key, common.CopyBytes(value)})
+func (b *memBatch) Put(key, value []byte) error {
+ b.lock.Lock()
+ defer b.lock.Unlock()
+
+ b.writes = append(b.writes, kv{key, common.CopyBytes(value)})
return nil
}
-func (w *memBatch) Write() error {
- for _, kv := range w.writes {
- w.db.db[string(kv.k)] = kv.v
+func (b *memBatch) Write() error {
+ b.lock.RLock()
+ defer b.lock.RUnlock()
+
+ b.db.lock.Lock()
+ defer b.db.lock.Unlock()
+
+ for _, kv := range b.writes {
+ b.db.db[string(kv.k)] = kv.v
}
return nil
}
diff --git a/miner/worker.go b/miner/worker.go
index d827cb97d..3519e1506 100644
--- a/miner/worker.go
+++ b/miner/worker.go
@@ -312,7 +312,7 @@ func (self *worker) wait() {
self.mux.Post(core.ChainHeadEvent{block})
self.mux.Post(logs)
}
- if err := core.PutBlockReceipts(self.chainDb, block, receipts); err != nil {
+ if err := core.PutBlockReceipts(self.chainDb, block.Hash(), receipts); err != nil {
glog.V(logger.Warn).Infoln("error writing block receipts:", err)
}
}(block, work.state.Logs(), work.receipts)
diff --git a/rpc/api/debug.go b/rpc/api/debug.go
index 003b4d994..d2cbc7f19 100644
--- a/rpc/api/debug.go
+++ b/rpc/api/debug.go
@@ -146,13 +146,7 @@ func (self *debugApi) SetHead(req *shared.Request) (interface{}, error) {
if err := self.codec.Decode(req.Params, &args); err != nil {
return nil, shared.NewDecodeParamError(err.Error())
}
-
- block := self.xeth.EthBlockByNumber(args.BlockNumber)
- if block == nil {
- return nil, fmt.Errorf("block #%d not found", args.BlockNumber)
- }
-
- self.ethereum.BlockChain().SetHead(block)
+ self.ethereum.BlockChain().SetHead(uint64(args.BlockNumber))
return nil, nil
}
diff --git a/rpc/api/eth.go b/rpc/api/eth.go
index 6db006a46..4722682ff 100644
--- a/rpc/api/eth.go
+++ b/rpc/api/eth.go
@@ -168,9 +168,7 @@ func (self *ethApi) IsMining(req *shared.Request) (interface{}, error) {
}
func (self *ethApi) IsSyncing(req *shared.Request) (interface{}, error) {
- current := self.ethereum.BlockChain().CurrentBlock().NumberU64()
- origin, height := self.ethereum.Downloader().Boundaries()
-
+ origin, current, height := self.ethereum.Downloader().Progress()
if current < height {
return map[string]interface{}{
"startingBlock": newHexNum(big.NewInt(int64(origin)).Bytes()),
diff --git a/rpc/api/eth_args.go b/rpc/api/eth_args.go
index 66c190a51..6aca6a663 100644
--- a/rpc/api/eth_args.go
+++ b/rpc/api/eth_args.go
@@ -838,7 +838,7 @@ func NewLogRes(log *vm.Log) LogRes {
}
l.Address = newHexData(log.Address)
l.Data = newHexData(log.Data)
- l.BlockNumber = newHexNum(log.Number)
+ l.BlockNumber = newHexNum(log.BlockNumber)
l.LogIndex = newHexNum(log.Index)
l.TransactionHash = newHexData(log.TxHash)
l.TransactionIndex = newHexNum(log.TxIndex)
diff --git a/rpc/api/parsing.go b/rpc/api/parsing.go
index cdfaa0ed1..7667616ff 100644
--- a/rpc/api/parsing.go
+++ b/rpc/api/parsing.go
@@ -453,8 +453,8 @@ func NewReceiptRes(rec *types.Receipt) *ReceiptRes {
v.ContractAddress = newHexData(rec.ContractAddress)
}
- logs := make([]interface{}, len(rec.Logs()))
- for i, log := range rec.Logs() {
+ logs := make([]interface{}, len(rec.Logs))
+ for i, log := range rec.Logs {
logs[i] = NewLogRes(log)
}
v.Logs = &logs
diff --git a/trie/sync.go b/trie/sync.go
new file mode 100644
index 000000000..d55399d06
--- /dev/null
+++ b/trie/sync.go
@@ -0,0 +1,285 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import (
+ "fmt"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+ "gopkg.in/karalabe/cookiejar.v2/collections/prque"
+)
+
+// request represents a scheduled or already in-flight state retrieval request.
+type request struct {
+ hash common.Hash // Hash of the node data content to retrieve
+ data []byte // Data content of the node, cached until all subtrees complete
+ object *node // Target node to populate with retrieved data (hashnode originally)
+
+ parents []*request // Parent state nodes referencing this entry (notify all upon completion)
+ depth int // Depth level within the trie the node is located to prioritise DFS
+ deps int // Number of dependencies before allowed to commit this node
+
+ callback TrieSyncLeafCallback // Callback to invoke if a leaf node it reached on this branch
+}
+
+// SyncResult is a simple list to return missing nodes along with their request
+// hashes.
+type SyncResult struct {
+ Hash common.Hash // Hash of the originally unknown trie node
+ Data []byte // Data content of the retrieved node
+}
+
+// TrieSyncLeafCallback is a callback type invoked when a trie sync reaches a
+// leaf node. It's used by state syncing to check if the leaf node requires some
+// further data syncing.
+type TrieSyncLeafCallback func(leaf []byte, parent common.Hash) error
+
+// TrieSync is the main state trie synchronisation scheduler, which provides yet
+// unknown trie hashes to retrieve, accepts node data associated with said hashes
+// and reconstructs the trie step by step until all is done.
+type TrieSync struct {
+ database ethdb.Database // State database for storing all the assembled node data
+ requests map[common.Hash]*request // Pending requests pertaining to a key hash
+ queue *prque.Prque // Priority queue with the pending requests
+}
+
+// NewTrieSync creates a new trie data download scheduler.
+func NewTrieSync(root common.Hash, database ethdb.Database, callback TrieSyncLeafCallback) *TrieSync {
+ ts := &TrieSync{
+ database: database,
+ requests: make(map[common.Hash]*request),
+ queue: prque.New(),
+ }
+ ts.AddSubTrie(root, 0, common.Hash{}, callback)
+ return ts
+}
+
+// AddSubTrie registers a new trie to the sync code, rooted at the designated parent.
+func (s *TrieSync) AddSubTrie(root common.Hash, depth int, parent common.Hash, callback TrieSyncLeafCallback) {
+ // Short circuit if the trie is empty or already known
+ if root == emptyRoot {
+ return
+ }
+ blob, _ := s.database.Get(root.Bytes())
+ if local, err := decodeNode(blob); local != nil && err == nil {
+ return
+ }
+ // Assemble the new sub-trie sync request
+ node := node(hashNode(root.Bytes()))
+ req := &request{
+ object: &node,
+ hash: root,
+ depth: depth,
+ callback: callback,
+ }
+ // If this sub-trie has a designated parent, link them together
+ if parent != (common.Hash{}) {
+ ancestor := s.requests[parent]
+ if ancestor == nil {
+ panic(fmt.Sprintf("sub-trie ancestor not found: %x", parent))
+ }
+ ancestor.deps++
+ req.parents = append(req.parents, ancestor)
+ }
+ s.schedule(req)
+}
+
+// AddRawEntry schedules the direct retrieval of a state entry that should not be
+// interpreted as a trie node, but rather accepted and stored into the database
+// as is. This method's goal is to support misc state metadata retrievals (e.g.
+// contract code).
+func (s *TrieSync) AddRawEntry(hash common.Hash, depth int, parent common.Hash) {
+ // Short circuit if the entry is empty or already known
+ if hash == emptyState {
+ return
+ }
+ if blob, _ := s.database.Get(hash.Bytes()); blob != nil {
+ return
+ }
+ // Assemble the new sub-trie sync request
+ req := &request{
+ hash: hash,
+ depth: depth,
+ }
+ // If this sub-trie has a designated parent, link them together
+ if parent != (common.Hash{}) {
+ ancestor := s.requests[parent]
+ if ancestor == nil {
+ panic(fmt.Sprintf("raw-entry ancestor not found: %x", parent))
+ }
+ ancestor.deps++
+ req.parents = append(req.parents, ancestor)
+ }
+ s.schedule(req)
+}
+
+// Missing retrieves the known missing nodes from the trie for retrieval.
+func (s *TrieSync) Missing(max int) []common.Hash {
+ requests := []common.Hash{}
+ for !s.queue.Empty() && (max == 0 || len(requests) < max) {
+ requests = append(requests, s.queue.PopItem().(common.Hash))
+ }
+ return requests
+}
+
+// Process injects a batch of retrieved trie nodes data.
+func (s *TrieSync) Process(results []SyncResult) (int, error) {
+ for i, item := range results {
+ // If the item was not requested, bail out
+ request := s.requests[item.Hash]
+ if request == nil {
+ return i, fmt.Errorf("not requested: %x", item.Hash)
+ }
+ // If the item is a raw entry request, commit directly
+ if request.object == nil {
+ request.data = item.Data
+ s.commit(request, nil)
+ continue
+ }
+ // Decode the node data content and update the request
+ node, err := decodeNode(item.Data)
+ if err != nil {
+ return i, err
+ }
+ *request.object = node
+ request.data = item.Data
+
+ // Create and schedule a request for all the children nodes
+ requests, err := s.children(request)
+ if err != nil {
+ return i, err
+ }
+ if len(requests) == 0 && request.deps == 0 {
+ s.commit(request, nil)
+ continue
+ }
+ request.deps += len(requests)
+ for _, child := range requests {
+ s.schedule(child)
+ }
+ }
+ return 0, nil
+}
+
+// Pending returns the number of state entries currently pending for download.
+func (s *TrieSync) Pending() int {
+ return len(s.requests)
+}
+
+// schedule inserts a new state retrieval request into the fetch queue. If there
+// is already a pending request for this node, the new request will be discarded
+// and only a parent reference added to the old one.
+func (s *TrieSync) schedule(req *request) {
+ // If we're already requesting this node, add a new reference and stop
+ if old, ok := s.requests[req.hash]; ok {
+ old.parents = append(old.parents, req.parents...)
+ return
+ }
+ // Schedule the request for future retrieval
+ s.queue.Push(req.hash, float32(req.depth))
+ s.requests[req.hash] = req
+}
+
+// children retrieves all the missing children of a state trie entry for future
+// retrieval scheduling.
+func (s *TrieSync) children(req *request) ([]*request, error) {
+ // Gather all the children of the node, irrelevant whether known or not
+ type child struct {
+ node *node
+ depth int
+ }
+ children := []child{}
+
+ switch node := (*req.object).(type) {
+ case shortNode:
+ children = []child{{
+ node: &node.Val,
+ depth: req.depth + len(node.Key),
+ }}
+ case fullNode:
+ for i := 0; i < 17; i++ {
+ if node[i] != nil {
+ children = append(children, child{
+ node: &node[i],
+ depth: req.depth + 1,
+ })
+ }
+ }
+ default:
+ panic(fmt.Sprintf("unknown node: %+v", node))
+ }
+ // Iterate over the children, and request all unknown ones
+ requests := make([]*request, 0, len(children))
+ for _, child := range children {
+ // Notify any external watcher of a new key/value node
+ if req.callback != nil {
+ if node, ok := (*child.node).(valueNode); ok {
+ if err := req.callback(node, req.hash); err != nil {
+ return nil, err
+ }
+ }
+ }
+ // If the child references another node, resolve or schedule
+ if node, ok := (*child.node).(hashNode); ok {
+ // Try to resolve the node from the local database
+ blob, _ := s.database.Get(node)
+ if local, err := decodeNode(blob); local != nil && err == nil {
+ *child.node = local
+ continue
+ }
+ // Locally unknown node, schedule for retrieval
+ requests = append(requests, &request{
+ object: child.node,
+ hash: common.BytesToHash(node),
+ parents: []*request{req},
+ depth: child.depth,
+ callback: req.callback,
+ })
+ }
+ }
+ return requests, nil
+}
+
+// commit finalizes a retrieval request and stores it into the database. If any
+// of the referencing parent requests complete due to this commit, they are also
+// committed themselves.
+func (s *TrieSync) commit(req *request, batch ethdb.Batch) (err error) {
+ // Create a new batch if none was specified
+ if batch == nil {
+ batch = s.database.NewBatch()
+ defer func() {
+ err = batch.Write()
+ }()
+ }
+ // Write the node content to disk
+ if err := batch.Put(req.hash[:], req.data); err != nil {
+ return err
+ }
+ delete(s.requests, req.hash)
+
+ // Check all parents for completion
+ for _, parent := range req.parents {
+ parent.deps--
+ if parent.deps == 0 {
+ if err := s.commit(parent, batch); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
diff --git a/trie/sync_test.go b/trie/sync_test.go
new file mode 100644
index 000000000..9c036a3a9
--- /dev/null
+++ b/trie/sync_test.go
@@ -0,0 +1,257 @@
+// Copyright 2015 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see .
+
+package trie
+
+import (
+ "bytes"
+ "testing"
+
+ "github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/ethdb"
+)
+
+// makeTestTrie create a sample test trie to test node-wise reconstruction.
+func makeTestTrie() (ethdb.Database, *Trie, map[string][]byte) {
+ // Create an empty trie
+ db, _ := ethdb.NewMemDatabase()
+ trie, _ := New(common.Hash{}, db)
+
+ // Fill it with some arbitrary data
+ content := make(map[string][]byte)
+ for i := byte(0); i < 255; i++ {
+ key, val := common.LeftPadBytes([]byte{1, i}, 32), []byte{i}
+ content[string(key)] = val
+ trie.Update(key, val)
+
+ key, val = common.LeftPadBytes([]byte{2, i}, 32), []byte{i}
+ content[string(key)] = val
+ trie.Update(key, val)
+ }
+ trie.Commit()
+
+ // Return the generated trie
+ return db, trie, content
+}
+
+// checkTrieContents cross references a reconstructed trie with an expected data
+// content map.
+func checkTrieContents(t *testing.T, db Database, root []byte, content map[string][]byte) {
+ trie, err := New(common.BytesToHash(root), db)
+ if err != nil {
+ t.Fatalf("failed to create trie at %x: %v", root, err)
+ }
+ for key, val := range content {
+ if have := trie.Get([]byte(key)); bytes.Compare(have, val) != 0 {
+ t.Errorf("entry %x: content mismatch: have %x, want %x", key, have, val)
+ }
+ }
+}
+
+// Tests that an empty trie is not scheduled for syncing.
+func TestEmptyTrieSync(t *testing.T) {
+ emptyA, _ := New(common.Hash{}, nil)
+ emptyB, _ := New(emptyRoot, nil)
+
+ for i, trie := range []*Trie{emptyA, emptyB} {
+ db, _ := ethdb.NewMemDatabase()
+ if req := NewTrieSync(common.BytesToHash(trie.Root()), db, nil).Missing(1); len(req) != 0 {
+ t.Errorf("test %d: content requested for empty trie: %v", i, req)
+ }
+ }
+}
+
+// Tests that given a root hash, a trie can sync iteratively on a single thread,
+// requesting retrieval tasks and returning all of them in one go.
+func TestIterativeTrieSyncIndividual(t *testing.T) { testIterativeTrieSync(t, 1) }
+func TestIterativeTrieSyncBatched(t *testing.T) { testIterativeTrieSync(t, 100) }
+
+func testIterativeTrieSync(t *testing.T, batch int) {
+ // Create a random trie to copy
+ srcDb, srcTrie, srcData := makeTestTrie()
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+
+ queue := append([]common.Hash{}, sched.Missing(batch)...)
+ for len(queue) > 0 {
+ results := make([]SyncResult, len(queue))
+ for i, hash := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = SyncResult{hash, data}
+ }
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = append(queue[:0], sched.Missing(batch)...)
+ }
+ // Cross check that the two tries re in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+}
+
+// Tests that the trie scheduler can correctly reconstruct the state even if only
+// partial results are returned, and the others sent only later.
+func TestIterativeDelayedTrieSync(t *testing.T) {
+ // Create a random trie to copy
+ srcDb, srcTrie, srcData := makeTestTrie()
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+
+ queue := append([]common.Hash{}, sched.Missing(10000)...)
+ for len(queue) > 0 {
+ // Sync only half of the scheduled nodes
+ results := make([]SyncResult, len(queue)/2+1)
+ for i, hash := range queue[:len(results)] {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results[i] = SyncResult{hash, data}
+ }
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = append(queue[len(results):], sched.Missing(10000)...)
+ }
+ // Cross check that the two tries re in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+}
+
+// Tests that given a root hash, a trie can sync iteratively on a single thread,
+// requesting retrieval tasks and returning all of them in one go, however in a
+// random order.
+func TestIterativeRandomTrieSyncIndividual(t *testing.T) { testIterativeRandomTrieSync(t, 1) }
+func TestIterativeRandomTrieSyncBatched(t *testing.T) { testIterativeRandomTrieSync(t, 100) }
+
+func testIterativeRandomTrieSync(t *testing.T, batch int) {
+ // Create a random trie to copy
+ srcDb, srcTrie, srcData := makeTestTrie()
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+
+ queue := make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(batch) {
+ queue[hash] = struct{}{}
+ }
+ for len(queue) > 0 {
+ // Fetch all the queued nodes in a random order
+ results := make([]SyncResult, 0, len(queue))
+ for hash, _ := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results = append(results, SyncResult{hash, data})
+ }
+ // Feed the retrieved results back and queue new tasks
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(batch) {
+ queue[hash] = struct{}{}
+ }
+ }
+ // Cross check that the two tries re in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+}
+
+// Tests that the trie scheduler can correctly reconstruct the state even if only
+// partial results are returned (Even those randomly), others sent only later.
+func TestIterativeRandomDelayedTrieSync(t *testing.T) {
+ // Create a random trie to copy
+ srcDb, srcTrie, srcData := makeTestTrie()
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+
+ queue := make(map[common.Hash]struct{})
+ for _, hash := range sched.Missing(10000) {
+ queue[hash] = struct{}{}
+ }
+ for len(queue) > 0 {
+ // Sync only half of the scheduled nodes, even those in random order
+ results := make([]SyncResult, 0, len(queue)/2+1)
+ for hash, _ := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ results = append(results, SyncResult{hash, data})
+
+ if len(results) >= cap(results) {
+ break
+ }
+ }
+ // Feed the retrieved results back and queue new tasks
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ for _, result := range results {
+ delete(queue, result.Hash)
+ }
+ for _, hash := range sched.Missing(10000) {
+ queue[hash] = struct{}{}
+ }
+ }
+ // Cross check that the two tries re in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+}
+
+// Tests that a trie sync will not request nodes multiple times, even if they
+// have such references.
+func TestDuplicateAvoidanceTrieSync(t *testing.T) {
+ // Create a random trie to copy
+ srcDb, srcTrie, srcData := makeTestTrie()
+
+ // Create a destination trie and sync with the scheduler
+ dstDb, _ := ethdb.NewMemDatabase()
+ sched := NewTrieSync(common.BytesToHash(srcTrie.Root()), dstDb, nil)
+
+ queue := append([]common.Hash{}, sched.Missing(0)...)
+ requested := make(map[common.Hash]struct{})
+
+ for len(queue) > 0 {
+ results := make([]SyncResult, len(queue))
+ for i, hash := range queue {
+ data, err := srcDb.Get(hash.Bytes())
+ if err != nil {
+ t.Fatalf("failed to retrieve node data for %x: %v", hash, err)
+ }
+ if _, ok := requested[hash]; ok {
+ t.Errorf("hash %x already requested once", hash)
+ }
+ requested[hash] = struct{}{}
+
+ results[i] = SyncResult{hash, data}
+ }
+ if index, err := sched.Process(results); err != nil {
+ t.Fatalf("failed to process result #%d: %v", index, err)
+ }
+ queue = append(queue[:0], sched.Missing(0)...)
+ }
+ // Cross check that the two tries re in sync
+ checkTrieContents(t, dstDb, srcTrie.Root(), srcData)
+}
diff --git a/trie/trie.go b/trie/trie.go
index aa8d39fe2..a3a383fb5 100644
--- a/trie/trie.go
+++ b/trie/trie.go
@@ -24,6 +24,7 @@ import (
"hash"
"github.com/ethereum/go-ethereum/common"
+ "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/crypto/sha3"
"github.com/ethereum/go-ethereum/logger"
"github.com/ethereum/go-ethereum/logger/glog"
@@ -35,8 +36,12 @@ const defaultCacheCapacity = 800
var (
// The global cache stores decoded trie nodes by hash as they get loaded.
globalCache = newARC(defaultCacheCapacity)
+
// This is the known root hash of an empty trie.
emptyRoot = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+
+ // This is the known hash of an empty state trie entry.
+ emptyState = crypto.Sha3Hash(nil)
)
var ErrMissingRoot = errors.New("missing root node")