forked from cerc-io/plugeth
eth: port the synchronisation algo to eth/62
This commit is contained in:
parent
ca88e18f59
commit
47a7fe5d22
@ -140,7 +140,7 @@ var (
|
||||
}
|
||||
EthVersionFlag = cli.IntFlag{
|
||||
Name: "eth",
|
||||
Value: 61,
|
||||
Value: 62,
|
||||
Usage: "Highest eth protocol to advertise (temporary, dev option)",
|
||||
}
|
||||
|
||||
|
@ -357,6 +357,20 @@ func (b *Block) WithMiningResult(nonce uint64, mixDigest common.Hash) *Block {
|
||||
}
|
||||
}
|
||||
|
||||
// WithBody returns a new block with the given transaction and uncle contents.
|
||||
func (b *Block) WithBody(transactions []*Transaction, uncles []*Header) *Block {
|
||||
block := &Block{
|
||||
header: copyHeader(b.header),
|
||||
transactions: make([]*Transaction, len(transactions)),
|
||||
uncles: make([]*Header, len(uncles)),
|
||||
}
|
||||
copy(block.transactions, transactions)
|
||||
for i := range uncles {
|
||||
block.uncles[i] = copyHeader(uncles[i])
|
||||
}
|
||||
return block
|
||||
}
|
||||
|
||||
// Implement pow.Block
|
||||
|
||||
func (b *Block) Hash() common.Hash {
|
||||
|
@ -26,12 +26,10 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/event"
|
||||
"github.com/ethereum/go-ethereum/logger"
|
||||
"github.com/ethereum/go-ethereum/logger/glog"
|
||||
"gopkg.in/fatih/set.v0"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -40,40 +38,44 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
MinHashFetch = 512 // Minimum amount of hashes to not consider a peer stalling
|
||||
MaxHashFetch = 512 // Amount of hashes to be fetched per retrieval request
|
||||
MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request
|
||||
MaxHeaderFetch = 256 // Amount of block headers to be fetched per retrieval request
|
||||
MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request
|
||||
MaxBodyFetch = 128 // Amount of block bodies to be fetched per retrieval request
|
||||
MaxStateFetch = 384 // Amount of node state values to allow fetching per request
|
||||
MaxReceiptsFetch = 384 // Amount of transaction receipts to allow fetching per request
|
||||
|
||||
hashTTL = 5 * time.Second // Time it takes for a hash request to time out
|
||||
blockSoftTTL = 3 * time.Second // Request completion threshold for increasing or decreasing a peer's bandwidth
|
||||
blockHardTTL = 3 * blockSoftTTL // Maximum time allowance before a block request is considered expired
|
||||
crossCheckCycle = time.Second // Period after which to check for expired cross checks
|
||||
hashTTL = 5 * time.Second // [eth/61] Time it takes for a hash request to time out
|
||||
blockSoftTTL = 3 * time.Second // [eth/61] Request completion threshold for increasing or decreasing a peer's bandwidth
|
||||
blockHardTTL = 3 * blockSoftTTL // [eth/61] Maximum time allowance before a block request is considered expired
|
||||
headerTTL = 5 * time.Second // [eth/62] Time it takes for a header request to time out
|
||||
bodySoftTTL = 3 * time.Second // [eth/62] Request completion threshold for increasing or decreasing a peer's bandwidth
|
||||
bodyHardTTL = 3 * bodySoftTTL // [eth/62] Maximum time allowance before a block body request is considered expired
|
||||
|
||||
maxQueuedHashes = 256 * 1024 // Maximum number of hashes to queue for import (DOS protection)
|
||||
maxBannedHashes = 4096 // Number of bannable hashes before phasing old ones out
|
||||
maxBlockProcess = 256 // Number of blocks to import at once into the chain
|
||||
maxQueuedHashes = 256 * 1024 // [eth/61] Maximum number of hashes to queue for import (DOS protection)
|
||||
maxQueuedHeaders = 256 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection)
|
||||
maxBlockProcess = 256 // Number of blocks to import at once into the chain
|
||||
)
|
||||
|
||||
var (
|
||||
errBusy = errors.New("busy")
|
||||
errUnknownPeer = errors.New("peer is unknown or unhealthy")
|
||||
errBadPeer = errors.New("action from bad peer ignored")
|
||||
errStallingPeer = errors.New("peer is stalling")
|
||||
errBannedHead = errors.New("peer head hash already banned")
|
||||
errNoPeers = errors.New("no peers to keep download active")
|
||||
errPendingQueue = errors.New("pending items in queue")
|
||||
errTimeout = errors.New("timeout")
|
||||
errEmptyHashSet = errors.New("empty hash set by peer")
|
||||
errPeersUnavailable = errors.New("no peers available or all peers tried for block download process")
|
||||
errAlreadyInPool = errors.New("hash already in pool")
|
||||
errInvalidChain = errors.New("retrieved hash chain is invalid")
|
||||
errCrossCheckFailed = errors.New("block cross-check failed")
|
||||
errCancelHashFetch = errors.New("hash fetching canceled (requested)")
|
||||
errCancelBlockFetch = errors.New("block downloading canceled (requested)")
|
||||
errNoSyncActive = errors.New("no sync active")
|
||||
errBusy = errors.New("busy")
|
||||
errUnknownPeer = errors.New("peer is unknown or unhealthy")
|
||||
errBadPeer = errors.New("action from bad peer ignored")
|
||||
errStallingPeer = errors.New("peer is stalling")
|
||||
errNoPeers = errors.New("no peers to keep download active")
|
||||
errPendingQueue = errors.New("pending items in queue")
|
||||
errTimeout = errors.New("timeout")
|
||||
errEmptyHashSet = errors.New("empty hash set by peer")
|
||||
errEmptyHeaderSet = errors.New("empty header set by peer")
|
||||
errPeersUnavailable = errors.New("no peers available or all peers tried for block download process")
|
||||
errAlreadyInPool = errors.New("hash already in pool")
|
||||
errInvalidChain = errors.New("retrieved hash chain is invalid")
|
||||
errInvalidBody = errors.New("retrieved block body is invalid")
|
||||
errCancelHashFetch = errors.New("hash fetching canceled (requested)")
|
||||
errCancelBlockFetch = errors.New("block downloading canceled (requested)")
|
||||
errCancelHeaderFetch = errors.New("block header fetching canceled (requested)")
|
||||
errCancelBodyFetch = errors.New("block body downloading canceled (requested)")
|
||||
errNoSyncActive = errors.New("no sync active")
|
||||
)
|
||||
|
||||
// hashCheckFn is a callback type for verifying a hash's presence in the local chain.
|
||||
@ -91,28 +93,36 @@ type chainInsertFn func(types.Blocks) (int, error)
|
||||
// peerDropFn is a callback type for dropping a peer detected as malicious.
|
||||
type peerDropFn func(id string)
|
||||
|
||||
type blockPack struct {
|
||||
peerId string
|
||||
blocks []*types.Block
|
||||
}
|
||||
|
||||
// hashPack is a batch of block hashes returned by a peer (eth/61).
|
||||
type hashPack struct {
|
||||
peerId string
|
||||
hashes []common.Hash
|
||||
}
|
||||
|
||||
type crossCheck struct {
|
||||
expire time.Time
|
||||
parent common.Hash
|
||||
// blockPack is a batch of blocks returned by a peer (eth/61).
|
||||
type blockPack struct {
|
||||
peerId string
|
||||
blocks []*types.Block
|
||||
}
|
||||
|
||||
// headerPack is a batch of block headers returned by a peer.
|
||||
type headerPack struct {
|
||||
peerId string
|
||||
headers []*types.Header
|
||||
}
|
||||
|
||||
// bodyPack is a batch of block bodies returned by a peer.
|
||||
type bodyPack struct {
|
||||
peerId string
|
||||
transactions [][]*types.Transaction
|
||||
uncles [][]*types.Header
|
||||
}
|
||||
|
||||
type Downloader struct {
|
||||
mux *event.TypeMux
|
||||
|
||||
queue *queue // Scheduler for selecting the hashes to download
|
||||
peers *peerSet // Set of active peers from which download can proceed
|
||||
checks map[common.Hash]*crossCheck // Pending cross checks to verify a hash chain
|
||||
banned *set.Set // Set of hashes we've received and banned
|
||||
queue *queue // Scheduler for selecting the hashes to download
|
||||
peers *peerSet // Set of active peers from which download can proceed
|
||||
|
||||
interrupt int32 // Atomic boolean to signal termination
|
||||
|
||||
@ -137,12 +147,18 @@ type Downloader struct {
|
||||
|
||||
// Channels
|
||||
newPeerCh chan *peer
|
||||
hashCh chan hashPack // Channel receiving inbound hashes
|
||||
blockCh chan blockPack // Channel receiving inbound blocks
|
||||
processCh chan bool // Channel to signal the block fetcher of new or finished work
|
||||
hashCh chan hashPack // [eth/61] Channel receiving inbound hashes
|
||||
blockCh chan blockPack // [eth/61] Channel receiving inbound blocks
|
||||
headerCh chan headerPack // [eth/62] Channel receiving inbound block headers
|
||||
bodyCh chan bodyPack // [eth/62] Channel receiving inbound block bodies
|
||||
processCh chan bool // Channel to signal the block fetcher of new or finished work
|
||||
|
||||
cancelCh chan struct{} // Channel to cancel mid-flight syncs
|
||||
cancelLock sync.RWMutex // Lock to protect the cancel channel in delivers
|
||||
|
||||
// Testing hooks
|
||||
bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch
|
||||
chainInsertHook func([]*Block) // Method to call upon inserting a chain of blocks (possibly in multiple invocations)
|
||||
}
|
||||
|
||||
// Block is an origin-tagged blockchain block.
|
||||
@ -153,8 +169,7 @@ type Block struct {
|
||||
|
||||
// New creates a new downloader to fetch hashes and blocks from remote peers.
|
||||
func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, headBlock headRetrievalFn, insertChain chainInsertFn, dropPeer peerDropFn) *Downloader {
|
||||
// Create the base downloader
|
||||
downloader := &Downloader{
|
||||
return &Downloader{
|
||||
mux: mux,
|
||||
queue: newQueue(),
|
||||
peers: newPeerSet(),
|
||||
@ -166,14 +181,10 @@ func New(mux *event.TypeMux, hasBlock hashCheckFn, getBlock blockRetrievalFn, he
|
||||
newPeerCh: make(chan *peer, 1),
|
||||
hashCh: make(chan hashPack, 1),
|
||||
blockCh: make(chan blockPack, 1),
|
||||
headerCh: make(chan headerPack, 1),
|
||||
bodyCh: make(chan bodyPack, 1),
|
||||
processCh: make(chan bool, 1),
|
||||
}
|
||||
// Inject all the known bad hashes
|
||||
downloader.banned = set.New()
|
||||
for hash, _ := range core.BadHashes {
|
||||
downloader.banned.Add(hash)
|
||||
}
|
||||
return downloader
|
||||
}
|
||||
|
||||
// Stats retrieves the current status of the downloader.
|
||||
@ -206,15 +217,12 @@ func (d *Downloader) Synchronising() bool {
|
||||
|
||||
// RegisterPeer injects a new download peer into the set of block source to be
|
||||
// used for fetching hashes and blocks from.
|
||||
func (d *Downloader) RegisterPeer(id string, version int, head common.Hash, getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn) error {
|
||||
// If the peer wants to send a banned hash, reject
|
||||
if d.banned.Has(head) {
|
||||
glog.V(logger.Debug).Infoln("Register rejected, head hash banned:", id)
|
||||
return errBannedHead
|
||||
}
|
||||
// Otherwise try to construct and register the peer
|
||||
func (d *Downloader) RegisterPeer(id string, version int, head common.Hash,
|
||||
getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading
|
||||
getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn) error {
|
||||
|
||||
glog.V(logger.Detail).Infoln("Registering peer", id)
|
||||
if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks)); err != nil {
|
||||
if err := d.peers.Register(newPeer(id, version, head, getRelHashes, getAbsHashes, getBlocks, getRelHeaders, getAbsHeaders, getBlockBodies)); err != nil {
|
||||
glog.V(logger.Error).Infoln("Register failed:", err)
|
||||
return err
|
||||
}
|
||||
@ -235,7 +243,7 @@ func (d *Downloader) UnregisterPeer(id string) error {
|
||||
// Synchronise tries to sync up our local block chain with a remote peer, both
|
||||
// adding various sanity checks as well as wrapping it with various log entries.
|
||||
func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
|
||||
glog.V(logger.Detail).Infof("Attempting synchronisation: %v, head 0x%x, TD %v", id, head[:4], td)
|
||||
glog.V(logger.Detail).Infof("Attempting synchronisation: %v, head [%x…], TD %v", id, head[:4], td)
|
||||
|
||||
switch err := d.synchronise(id, head, td); err {
|
||||
case nil:
|
||||
@ -244,7 +252,7 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int) {
|
||||
case errBusy:
|
||||
glog.V(logger.Detail).Infof("Synchronisation already in progress")
|
||||
|
||||
case errTimeout, errBadPeer, errStallingPeer, errBannedHead, errEmptyHashSet, errPeersUnavailable, errInvalidChain, errCrossCheckFailed:
|
||||
case errTimeout, errBadPeer, errStallingPeer, errEmptyHashSet, errEmptyHeaderSet, errPeersUnavailable, errInvalidChain:
|
||||
glog.V(logger.Debug).Infof("Removing peer %v: %v", id, err)
|
||||
d.dropPeer(id)
|
||||
|
||||
@ -270,10 +278,6 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
|
||||
}
|
||||
defer atomic.StoreInt32(&d.synchronising, 0)
|
||||
|
||||
// If the head hash is banned, terminate immediately
|
||||
if d.banned.Has(hash) {
|
||||
return errBannedHead
|
||||
}
|
||||
// Post a user notification of the sync (only once per session)
|
||||
if atomic.CompareAndSwapInt32(&d.notified, 0, 1) {
|
||||
glog.V(logger.Info).Infoln("Block synchronisation started")
|
||||
@ -285,7 +289,6 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int) error
|
||||
// Reset the queue and peer set to clean any internal leftover state
|
||||
d.queue.Reset()
|
||||
d.peers.Reset()
|
||||
d.checks = make(map[common.Hash]*crossCheck)
|
||||
|
||||
// Create cancel channel for aborting mid-flight
|
||||
d.cancelLock.Lock()
|
||||
@ -320,17 +323,37 @@ func (d *Downloader) syncWithPeer(p *peer, hash common.Hash, td *big.Int) (err e
|
||||
}
|
||||
}()
|
||||
|
||||
glog.V(logger.Debug).Infof("Synchronizing with the network using: %s, eth/%d", p.id, p.version)
|
||||
switch p.version {
|
||||
case eth61:
|
||||
glog.V(logger.Debug).Infof("Synchronising with the network using: %s [eth/%d]", p.id, p.version)
|
||||
defer glog.V(logger.Debug).Infof("Synchronisation terminated")
|
||||
|
||||
switch {
|
||||
case p.version == eth61:
|
||||
// Old eth/61, use forward, concurrent hash and block retrieval algorithm
|
||||
number, err := d.findAncestor61(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errc := make(chan error, 2)
|
||||
go func() { errc <- d.fetchHashes61(p, td, number+1) }()
|
||||
go func() { errc <- d.fetchBlocks61(number + 1) }()
|
||||
|
||||
// If any fetcher fails, cancel the other
|
||||
if err := <-errc; err != nil {
|
||||
d.cancel()
|
||||
<-errc
|
||||
return err
|
||||
}
|
||||
return <-errc
|
||||
|
||||
case p.version >= eth62:
|
||||
// New eth/62, use forward, concurrent header and block body retrieval algorithm
|
||||
number, err := d.findAncestor(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errc := make(chan error, 2)
|
||||
go func() { errc <- d.fetchHashes(p, td, number+1) }()
|
||||
go func() { errc <- d.fetchBlocks(number + 1) }()
|
||||
go func() { errc <- d.fetchHeaders(p, td, number+1) }()
|
||||
go func() { errc <- d.fetchBodies(number + 1) }()
|
||||
|
||||
// If any fetcher fails, cancel the other
|
||||
if err := <-errc; err != nil {
|
||||
@ -373,17 +396,17 @@ func (d *Downloader) Terminate() {
|
||||
d.cancel()
|
||||
}
|
||||
|
||||
// findAncestor tries to locate the common ancestor block of the local chain and
|
||||
// findAncestor61 tries to locate the common ancestor block of the local chain and
|
||||
// a remote peers blockchain. In the general case when our node was in sync and
|
||||
// on the correct chain, checking the top N blocks should already get us a match.
|
||||
// In the rare scenario when we ended up on a long soft fork (i.e. none of the
|
||||
// head blocks match), we do a binary search to find the common ancestor.
|
||||
func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
// In the rare scenario when we ended up on a long reorganization (i.e. none of
|
||||
// the head blocks match), we do a binary search to find the common ancestor.
|
||||
func (d *Downloader) findAncestor61(p *peer) (uint64, error) {
|
||||
glog.V(logger.Debug).Infof("%v: looking for common ancestor", p)
|
||||
|
||||
// Request out head blocks to short circuit ancestor location
|
||||
head := d.headBlock().NumberU64()
|
||||
from := int64(head) - int64(MaxHashFetch)
|
||||
from := int64(head) - int64(MaxHashFetch) + 1
|
||||
if from < 0 {
|
||||
from = 0
|
||||
}
|
||||
@ -422,6 +445,12 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
case <-d.blockCh:
|
||||
// Out of bounds blocks received, ignore them
|
||||
|
||||
case <-d.headerCh:
|
||||
// Out of bounds eth/62 block headers received, ignore them
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds eth/62 block bodies received, ignore them
|
||||
|
||||
case <-timeout:
|
||||
glog.V(logger.Debug).Infof("%v: head hash timeout", p)
|
||||
return 0, errTimeout
|
||||
@ -429,7 +458,7 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
}
|
||||
// If the head fetch already found an ancestor, return
|
||||
if !common.EmptyHash(hash) {
|
||||
glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x]", p, number, hash[:4])
|
||||
glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash[:4])
|
||||
return number, nil
|
||||
}
|
||||
// Ancestor not found, we need to binary search over our chain
|
||||
@ -468,7 +497,7 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
break
|
||||
}
|
||||
if block.NumberU64() != check {
|
||||
glog.V(logger.Debug).Infof("%v: non requested hash #%d [%x], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check)
|
||||
glog.V(logger.Debug).Infof("%v: non requested hash #%d [%x…], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check)
|
||||
return 0, errBadPeer
|
||||
}
|
||||
start = check
|
||||
@ -476,6 +505,12 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
case <-d.blockCh:
|
||||
// Out of bounds blocks received, ignore them
|
||||
|
||||
case <-d.headerCh:
|
||||
// Out of bounds eth/62 block headers received, ignore them
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds eth/62 block bodies received, ignore them
|
||||
|
||||
case <-timeout:
|
||||
glog.V(logger.Debug).Infof("%v: search hash timeout", p)
|
||||
return 0, errTimeout
|
||||
@ -485,9 +520,9 @@ func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
return start, nil
|
||||
}
|
||||
|
||||
// fetchHashes keeps retrieving hashes from the requested number, until no more
|
||||
// fetchHashes61 keeps retrieving hashes from the requested number, until no more
|
||||
// are returned, potentially throttling on the way.
|
||||
func (d *Downloader) fetchHashes(p *peer, td *big.Int, from uint64) error {
|
||||
func (d *Downloader) fetchHashes61(p *peer, td *big.Int, from uint64) error {
|
||||
glog.V(logger.Debug).Infof("%v: downloading hashes from #%d", p, from)
|
||||
|
||||
// Create a timeout timer, and the associated hash fetcher
|
||||
@ -510,6 +545,12 @@ func (d *Downloader) fetchHashes(p *peer, td *big.Int, from uint64) error {
|
||||
case <-d.cancelCh:
|
||||
return errCancelHashFetch
|
||||
|
||||
case <-d.headerCh:
|
||||
// Out of bounds eth/62 block headers received, ignore them
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds eth/62 block bodies received, ignore them
|
||||
|
||||
case hashPack := <-d.hashCh:
|
||||
// Make sure the active peer is giving us the hashes
|
||||
if hashPack.peerId != p.id {
|
||||
@ -548,7 +589,7 @@ func (d *Downloader) fetchHashes(p *peer, td *big.Int, from uint64) error {
|
||||
// Otherwise insert all the new hashes, aborting in case of junk
|
||||
glog.V(logger.Detail).Infof("%v: inserting %d hashes from #%d", p, len(hashPack.hashes), from)
|
||||
|
||||
inserts := d.queue.Insert(hashPack.hashes, true)
|
||||
inserts := d.queue.Insert61(hashPack.hashes, true)
|
||||
if len(inserts) != len(hashPack.hashes) {
|
||||
glog.V(logger.Debug).Infof("%v: stale hashes", p)
|
||||
return errBadPeer
|
||||
@ -573,10 +614,10 @@ func (d *Downloader) fetchHashes(p *peer, td *big.Int, from uint64) error {
|
||||
}
|
||||
}
|
||||
|
||||
// fetchBlocks iteratively downloads the scheduled hashes, taking any available
|
||||
// fetchBlocks61 iteratively downloads the scheduled hashes, taking any available
|
||||
// peers, reserving a chunk of blocks for each, waiting for delivery and also
|
||||
// periodically checking for timeouts.
|
||||
func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
func (d *Downloader) fetchBlocks61(from uint64) error {
|
||||
glog.V(logger.Debug).Infof("Downloading blocks from #%d", from)
|
||||
defer glog.V(logger.Debug).Infof("Block download terminated")
|
||||
|
||||
@ -595,24 +636,30 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
case <-d.cancelCh:
|
||||
return errCancelBlockFetch
|
||||
|
||||
case <-d.headerCh:
|
||||
// Out of bounds eth/62 block headers received, ignore them
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds eth/62 block bodies received, ignore them
|
||||
|
||||
case blockPack := <-d.blockCh:
|
||||
// If the peer was previously banned and failed to deliver it's pack
|
||||
// in a reasonable time frame, ignore it's message.
|
||||
if peer := d.peers.Peer(blockPack.peerId); peer != nil {
|
||||
// Deliver the received chunk of blocks, and demote in case of errors
|
||||
err := d.queue.Deliver(blockPack.peerId, blockPack.blocks)
|
||||
err := d.queue.Deliver61(blockPack.peerId, blockPack.blocks)
|
||||
switch err {
|
||||
case nil:
|
||||
// If no blocks were delivered, demote the peer (need the delivery above)
|
||||
if len(blockPack.blocks) == 0 {
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
peer.SetIdle61()
|
||||
glog.V(logger.Detail).Infof("%s: no blocks delivered", peer)
|
||||
break
|
||||
}
|
||||
// All was successful, promote the peer and potentially start processing
|
||||
peer.Promote()
|
||||
peer.SetIdle()
|
||||
peer.SetIdle61()
|
||||
glog.V(logger.Detail).Infof("%s: delivered %d blocks", peer, len(blockPack.blocks))
|
||||
go d.process()
|
||||
|
||||
@ -624,7 +671,7 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
// Peer probably timed out with its delivery but came through
|
||||
// in the end, demote, but allow to to pull from this peer.
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
peer.SetIdle61()
|
||||
glog.V(logger.Detail).Infof("%s: out of bound delivery", peer)
|
||||
|
||||
case errStaleDelivery:
|
||||
@ -638,7 +685,7 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
default:
|
||||
// Peer did something semi-useful, demote but keep it around
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
peer.SetIdle61()
|
||||
glog.V(logger.Detail).Infof("%s: delivery partially failed: %v", peer, err)
|
||||
go d.process()
|
||||
}
|
||||
@ -696,7 +743,7 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
// Reserve a chunk of hashes for a peer. A nil can mean either that
|
||||
// no more hashes are available, or that the peer is known not to
|
||||
// have them.
|
||||
request := d.queue.Reserve(peer, peer.Capacity())
|
||||
request := d.queue.Reserve61(peer, peer.Capacity())
|
||||
if request == nil {
|
||||
continue
|
||||
}
|
||||
@ -704,7 +751,7 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
glog.Infof("%s: requesting %d blocks", peer, len(request.Hashes))
|
||||
}
|
||||
// Fetch the chunk and make sure any errors return the hashes to the queue
|
||||
if err := peer.Fetch(request); err != nil {
|
||||
if err := peer.Fetch61(request); err != nil {
|
||||
glog.V(logger.Error).Infof("%v: fetch failed, rescheduling", peer)
|
||||
d.queue.Cancel(request)
|
||||
}
|
||||
@ -718,6 +765,401 @@ func (d *Downloader) fetchBlocks(from uint64) error {
|
||||
}
|
||||
}
|
||||
|
||||
// findAncestor tries to locate the common ancestor block of the local chain and
|
||||
// a remote peers blockchain. In the general case when our node was in sync and
|
||||
// on the correct chain, checking the top N blocks should already get us a match.
|
||||
// In the rare scenario when we ended up on a long reorganization (i.e. none of
|
||||
// the head blocks match), we do a binary search to find the common ancestor.
|
||||
func (d *Downloader) findAncestor(p *peer) (uint64, error) {
|
||||
glog.V(logger.Debug).Infof("%v: looking for common ancestor", p)
|
||||
|
||||
// Request our head blocks to short circuit ancestor location
|
||||
head := d.headBlock().NumberU64()
|
||||
from := int64(head) - int64(MaxHeaderFetch) + 1
|
||||
if from < 0 {
|
||||
from = 0
|
||||
}
|
||||
go p.getAbsHeaders(uint64(from), MaxHeaderFetch, 0, false)
|
||||
|
||||
// Wait for the remote response to the head fetch
|
||||
number, hash := uint64(0), common.Hash{}
|
||||
timeout := time.After(hashTTL)
|
||||
|
||||
for finished := false; !finished; {
|
||||
select {
|
||||
case <-d.cancelCh:
|
||||
return 0, errCancelHashFetch
|
||||
|
||||
case headerPack := <-d.headerCh:
|
||||
// Discard anything not from the origin peer
|
||||
if headerPack.peerId != p.id {
|
||||
glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
|
||||
break
|
||||
}
|
||||
// Make sure the peer actually gave something valid
|
||||
headers := headerPack.headers
|
||||
if len(headers) == 0 {
|
||||
glog.V(logger.Debug).Infof("%v: empty head header set", p)
|
||||
return 0, errEmptyHeaderSet
|
||||
}
|
||||
// Check if a common ancestor was found
|
||||
finished = true
|
||||
for i := len(headers) - 1; i >= 0; i-- {
|
||||
if d.hasBlock(headers[i].Hash()) {
|
||||
number, hash = headers[i].Number.Uint64(), headers[i].Hash()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds block bodies received, ignore them
|
||||
|
||||
case <-d.hashCh:
|
||||
// Out of bounds eth/61 hashes received, ignore them
|
||||
|
||||
case <-d.blockCh:
|
||||
// Out of bounds eth/61 blocks received, ignore them
|
||||
|
||||
case <-timeout:
|
||||
glog.V(logger.Debug).Infof("%v: head header timeout", p)
|
||||
return 0, errTimeout
|
||||
}
|
||||
}
|
||||
// If the head fetch already found an ancestor, return
|
||||
if !common.EmptyHash(hash) {
|
||||
glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash[:4])
|
||||
return number, nil
|
||||
}
|
||||
// Ancestor not found, we need to binary search over our chain
|
||||
start, end := uint64(0), head
|
||||
for start+1 < end {
|
||||
// Split our chain interval in two, and request the hash to cross check
|
||||
check := (start + end) / 2
|
||||
|
||||
timeout := time.After(hashTTL)
|
||||
go p.getAbsHeaders(uint64(check), 1, 0, false)
|
||||
|
||||
// Wait until a reply arrives to this request
|
||||
for arrived := false; !arrived; {
|
||||
select {
|
||||
case <-d.cancelCh:
|
||||
return 0, errCancelHashFetch
|
||||
|
||||
case headerPack := <-d.headerCh:
|
||||
// Discard anything not from the origin peer
|
||||
if headerPack.peerId != p.id {
|
||||
glog.V(logger.Debug).Infof("Received headers from incorrect peer(%s)", headerPack.peerId)
|
||||
break
|
||||
}
|
||||
// Make sure the peer actually gave something valid
|
||||
headers := headerPack.headers
|
||||
if len(headers) != 1 {
|
||||
glog.V(logger.Debug).Infof("%v: invalid search header set (%d)", p, len(headers))
|
||||
return 0, errBadPeer
|
||||
}
|
||||
arrived = true
|
||||
|
||||
// Modify the search interval based on the response
|
||||
block := d.getBlock(headers[0].Hash())
|
||||
if block == nil {
|
||||
end = check
|
||||
break
|
||||
}
|
||||
if block.NumberU64() != check {
|
||||
glog.V(logger.Debug).Infof("%v: non requested header #%d [%x…], instead of #%d", p, block.NumberU64(), block.Hash().Bytes()[:4], check)
|
||||
return 0, errBadPeer
|
||||
}
|
||||
start = check
|
||||
|
||||
case <-d.bodyCh:
|
||||
// Out of bounds block bodies received, ignore them
|
||||
|
||||
case <-d.hashCh:
|
||||
// Out of bounds eth/61 hashes received, ignore them
|
||||
|
||||
case <-d.blockCh:
|
||||
// Out of bounds eth/61 blocks received, ignore them
|
||||
|
||||
case <-timeout:
|
||||
glog.V(logger.Debug).Infof("%v: search header timeout", p)
|
||||
return 0, errTimeout
|
||||
}
|
||||
}
|
||||
}
|
||||
return start, nil
|
||||
}
|
||||
|
||||
// fetchHeaders keeps retrieving headers from the requested number, until no more
|
||||
// are returned, potentially throttling on the way.
|
||||
func (d *Downloader) fetchHeaders(p *peer, td *big.Int, from uint64) error {
|
||||
glog.V(logger.Debug).Infof("%v: downloading headers from #%d", p, from)
|
||||
defer glog.V(logger.Debug).Infof("%v: header download terminated", p)
|
||||
|
||||
// Create a timeout timer, and the associated hash fetcher
|
||||
timeout := time.NewTimer(0) // timer to dump a non-responsive active peer
|
||||
<-timeout.C // timeout channel should be initially empty
|
||||
defer timeout.Stop()
|
||||
|
||||
getHeaders := func(from uint64) {
|
||||
glog.V(logger.Detail).Infof("%v: fetching %d headers from #%d", p, MaxHeaderFetch, from)
|
||||
|
||||
go p.getAbsHeaders(from, MaxHeaderFetch, 0, false)
|
||||
timeout.Reset(headerTTL)
|
||||
}
|
||||
// Start pulling headers, until all are exhausted
|
||||
getHeaders(from)
|
||||
gotHeaders := false
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.cancelCh:
|
||||
return errCancelHeaderFetch
|
||||
|
||||
case <-d.hashCh:
|
||||
// Out of bounds eth/61 hashes received, ignore them
|
||||
|
||||
case <-d.blockCh:
|
||||
// Out of bounds eth/61 blocks received, ignore them
|
||||
|
||||
case headerPack := <-d.headerCh:
|
||||
// Make sure the active peer is giving us the headers
|
||||
if headerPack.peerId != p.id {
|
||||
glog.V(logger.Debug).Infof("Received headers from incorrect peer (%s)", headerPack.peerId)
|
||||
break
|
||||
}
|
||||
timeout.Stop()
|
||||
|
||||
// If no more headers are inbound, notify the body fetcher and return
|
||||
if len(headerPack.headers) == 0 {
|
||||
glog.V(logger.Debug).Infof("%v: no available headers", p)
|
||||
|
||||
select {
|
||||
case d.processCh <- false:
|
||||
case <-d.cancelCh:
|
||||
}
|
||||
// If no headers were retrieved at all, the peer violated it's TD promise that it had a
|
||||
// better chain compared to ours. The only exception is if it's promised blocks were
|
||||
// already imported by other means (e.g. fecher):
|
||||
//
|
||||
// R <remote peer>, L <local node>: Both at block 10
|
||||
// R: Mine block 11, and propagate it to L
|
||||
// L: Queue block 11 for import
|
||||
// L: Notice that R's head and TD increased compared to ours, start sync
|
||||
// L: Import of block 11 finishes
|
||||
// L: Sync begins, and finds common ancestor at 11
|
||||
// L: Request new headers up from 11 (R's TD was higher, it must have something)
|
||||
// R: Nothing to give
|
||||
if !gotHeaders && td.Cmp(d.headBlock().Td) > 0 {
|
||||
return errStallingPeer
|
||||
}
|
||||
return nil
|
||||
}
|
||||
gotHeaders = true
|
||||
|
||||
// Otherwise insert all the new headers, aborting in case of junk
|
||||
glog.V(logger.Detail).Infof("%v: inserting %d headers from #%d", p, len(headerPack.headers), from)
|
||||
|
||||
inserts := d.queue.Insert(headerPack.headers)
|
||||
if len(inserts) != len(headerPack.headers) {
|
||||
glog.V(logger.Debug).Infof("%v: stale headers", p)
|
||||
return errBadPeer
|
||||
}
|
||||
// Notify the block fetcher of new headers, but stop if queue is full
|
||||
cont := d.queue.Pending() < maxQueuedHeaders
|
||||
select {
|
||||
case d.processCh <- cont:
|
||||
default:
|
||||
}
|
||||
if !cont {
|
||||
return nil
|
||||
}
|
||||
// Queue not yet full, fetch the next batch
|
||||
from += uint64(len(headerPack.headers))
|
||||
getHeaders(from)
|
||||
|
||||
case <-timeout.C:
|
||||
// Header retrieval timed out, consider the peer bad and drop
|
||||
glog.V(logger.Debug).Infof("%v: header request timed out", p)
|
||||
d.dropPeer(p.id)
|
||||
|
||||
// Finish the sync gracefully instead of dumping the gathered data though
|
||||
select {
|
||||
case d.processCh <- false:
|
||||
default:
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// fetchBodies iteratively downloads the scheduled block bodies, taking any
|
||||
// available peers, reserving a chunk of blocks for each, waiting for delivery
|
||||
// and also periodically checking for timeouts.
|
||||
func (d *Downloader) fetchBodies(from uint64) error {
|
||||
glog.V(logger.Debug).Infof("Downloading block bodies from #%d", from)
|
||||
defer glog.V(logger.Debug).Infof("Block body download terminated")
|
||||
|
||||
// Create a timeout timer for scheduling expiration tasks
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
update := make(chan struct{}, 1)
|
||||
|
||||
// Prepare the queue and fetch block bodies until the block header fetcher's done
|
||||
d.queue.Prepare(from)
|
||||
finished := false
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-d.cancelCh:
|
||||
return errCancelBlockFetch
|
||||
|
||||
case <-d.hashCh:
|
||||
// Out of bounds eth/61 hashes received, ignore them
|
||||
|
||||
case <-d.blockCh:
|
||||
// Out of bounds eth/61 blocks received, ignore them
|
||||
|
||||
case bodyPack := <-d.bodyCh:
|
||||
// If the peer was previously banned and failed to deliver it's pack
|
||||
// in a reasonable time frame, ignore it's message.
|
||||
if peer := d.peers.Peer(bodyPack.peerId); peer != nil {
|
||||
// Deliver the received chunk of bodies, and demote in case of errors
|
||||
err := d.queue.Deliver(bodyPack.peerId, bodyPack.transactions, bodyPack.uncles)
|
||||
switch err {
|
||||
case nil:
|
||||
// If no blocks were delivered, demote the peer (need the delivery above)
|
||||
if len(bodyPack.transactions) == 0 || len(bodyPack.uncles) == 0 {
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
glog.V(logger.Detail).Infof("%s: no block bodies delivered", peer)
|
||||
break
|
||||
}
|
||||
// All was successful, promote the peer and potentially start processing
|
||||
peer.Promote()
|
||||
peer.SetIdle()
|
||||
glog.V(logger.Detail).Infof("%s: delivered %d:%d block bodies", peer, len(bodyPack.transactions), len(bodyPack.uncles))
|
||||
go d.process()
|
||||
|
||||
case errInvalidChain:
|
||||
// The hash chain is invalid (blocks are not ordered properly), abort
|
||||
return err
|
||||
|
||||
case errInvalidBody:
|
||||
// The peer delivered something very bad, drop immediately
|
||||
glog.V(logger.Error).Infof("%s: delivered invalid block, dropping", peer)
|
||||
d.dropPeer(peer.id)
|
||||
|
||||
case errNoFetchesPending:
|
||||
// Peer probably timed out with its delivery but came through
|
||||
// in the end, demote, but allow to to pull from this peer.
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
glog.V(logger.Detail).Infof("%s: out of bound delivery", peer)
|
||||
|
||||
case errStaleDelivery:
|
||||
// Delivered something completely else than requested, usually
|
||||
// caused by a timeout and delivery during a new sync cycle.
|
||||
// Don't set it to idle as the original request should still be
|
||||
// in flight.
|
||||
peer.Demote()
|
||||
glog.V(logger.Detail).Infof("%s: stale delivery", peer)
|
||||
|
||||
default:
|
||||
// Peer did something semi-useful, demote but keep it around
|
||||
peer.Demote()
|
||||
peer.SetIdle()
|
||||
glog.V(logger.Detail).Infof("%s: delivery partially failed: %v", peer, err)
|
||||
go d.process()
|
||||
}
|
||||
}
|
||||
// Blocks assembled, try to update the progress
|
||||
select {
|
||||
case update <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
case cont := <-d.processCh:
|
||||
// The header fetcher sent a continuation flag, check if it's done
|
||||
if !cont {
|
||||
finished = true
|
||||
}
|
||||
// Headers arrive, try to update the progress
|
||||
select {
|
||||
case update <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
case <-ticker.C:
|
||||
// Sanity check update the progress
|
||||
select {
|
||||
case update <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
|
||||
case <-update:
|
||||
// Short circuit if we lost all our peers
|
||||
if d.peers.Len() == 0 {
|
||||
return errNoPeers
|
||||
}
|
||||
// Check for block body request timeouts and demote the responsible peers
|
||||
for _, pid := range d.queue.Expire(bodyHardTTL) {
|
||||
if peer := d.peers.Peer(pid); peer != nil {
|
||||
peer.Demote()
|
||||
glog.V(logger.Detail).Infof("%s: block body delivery timeout", peer)
|
||||
}
|
||||
}
|
||||
// If there's noting more to fetch, wait or terminate
|
||||
if d.queue.Pending() == 0 {
|
||||
if d.queue.InFlight() == 0 && finished {
|
||||
glog.V(logger.Debug).Infof("Block body fetching completed")
|
||||
return nil
|
||||
}
|
||||
break
|
||||
}
|
||||
// Send a download request to all idle peers, until throttled
|
||||
queuedEmptyBlocks, throttled := false, false
|
||||
for _, peer := range d.peers.IdlePeers() {
|
||||
// Short circuit if throttling activated
|
||||
if d.queue.Throttle() {
|
||||
throttled = true
|
||||
break
|
||||
}
|
||||
// Reserve a chunk of hashes for a peer. A nil can mean either that
|
||||
// no more hashes are available, or that the peer is known not to
|
||||
// have them.
|
||||
request, process, err := d.queue.Reserve(peer, peer.Capacity())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if process {
|
||||
queuedEmptyBlocks = true
|
||||
go d.process()
|
||||
}
|
||||
if request == nil {
|
||||
continue
|
||||
}
|
||||
if glog.V(logger.Detail) {
|
||||
glog.Infof("%s: requesting %d block bodies", peer, len(request.Headers))
|
||||
}
|
||||
// Fetch the chunk and make sure any errors return the hashes to the queue
|
||||
if d.bodyFetchHook != nil {
|
||||
d.bodyFetchHook(request.Headers)
|
||||
}
|
||||
if err := peer.Fetch(request); err != nil {
|
||||
glog.V(logger.Error).Infof("%v: fetch failed, rescheduling", peer)
|
||||
d.queue.Cancel(request)
|
||||
}
|
||||
}
|
||||
// Make sure that we have peers available for fetching. If all peers have been tried
|
||||
// and all failed throw an error
|
||||
if !queuedEmptyBlocks && !throttled && d.queue.InFlight() == 0 {
|
||||
return errPeersUnavailable
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// process takes blocks from the queue and tries to import them into the chain.
|
||||
//
|
||||
// The algorithmic flow is as follows:
|
||||
@ -763,6 +1205,9 @@ func (d *Downloader) process() {
|
||||
if len(blocks) == 0 {
|
||||
return
|
||||
}
|
||||
if d.chainInsertHook != nil {
|
||||
d.chainInsertHook(blocks)
|
||||
}
|
||||
// Reset the import statistics
|
||||
d.importLock.Lock()
|
||||
d.importStart = time.Now()
|
||||
@ -796,9 +1241,31 @@ func (d *Downloader) process() {
|
||||
}
|
||||
}
|
||||
|
||||
// DeliverBlocks injects a new batch of blocks received from a remote node.
|
||||
// DeliverHashes61 injects a new batch of hashes received from a remote node into
|
||||
// the download schedule. This is usually invoked through the BlockHashesMsg by
|
||||
// the protocol handler.
|
||||
func (d *Downloader) DeliverHashes61(id string, hashes []common.Hash) error {
|
||||
// Make sure the downloader is active
|
||||
if atomic.LoadInt32(&d.synchronising) == 0 {
|
||||
return errNoSyncActive
|
||||
}
|
||||
// Deliver or abort if the sync is canceled while queuing
|
||||
d.cancelLock.RLock()
|
||||
cancel := d.cancelCh
|
||||
d.cancelLock.RUnlock()
|
||||
|
||||
select {
|
||||
case d.hashCh <- hashPack{id, hashes}:
|
||||
return nil
|
||||
|
||||
case <-cancel:
|
||||
return errNoSyncActive
|
||||
}
|
||||
}
|
||||
|
||||
// DeliverBlocks61 injects a new batch of blocks received from a remote node.
|
||||
// This is usually invoked through the BlocksMsg by the protocol handler.
|
||||
func (d *Downloader) DeliverBlocks(id string, blocks []*types.Block) error {
|
||||
func (d *Downloader) DeliverBlocks61(id string, blocks []*types.Block) error {
|
||||
// Make sure the downloader is active
|
||||
if atomic.LoadInt32(&d.synchronising) == 0 {
|
||||
return errNoSyncActive
|
||||
@ -817,10 +1284,9 @@ func (d *Downloader) DeliverBlocks(id string, blocks []*types.Block) error {
|
||||
}
|
||||
}
|
||||
|
||||
// DeliverHashes injects a new batch of hashes received from a remote node into
|
||||
// the download schedule. This is usually invoked through the BlockHashesMsg by
|
||||
// the protocol handler.
|
||||
func (d *Downloader) DeliverHashes(id string, hashes []common.Hash) error {
|
||||
// DeliverHeaders injects a new batch of blck headers received from a remote
|
||||
// node into the download schedule.
|
||||
func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) error {
|
||||
// Make sure the downloader is active
|
||||
if atomic.LoadInt32(&d.synchronising) == 0 {
|
||||
return errNoSyncActive
|
||||
@ -831,7 +1297,27 @@ func (d *Downloader) DeliverHashes(id string, hashes []common.Hash) error {
|
||||
d.cancelLock.RUnlock()
|
||||
|
||||
select {
|
||||
case d.hashCh <- hashPack{id, hashes}:
|
||||
case d.headerCh <- headerPack{id, headers}:
|
||||
return nil
|
||||
|
||||
case <-cancel:
|
||||
return errNoSyncActive
|
||||
}
|
||||
}
|
||||
|
||||
// DeliverBodies injects a new batch of block bodies received from a remote node.
|
||||
func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) error {
|
||||
// Make sure the downloader is active
|
||||
if atomic.LoadInt32(&d.synchronising) == 0 {
|
||||
return errNoSyncActive
|
||||
}
|
||||
// Deliver or abort if the sync is canceled while queuing
|
||||
d.cancelLock.RLock()
|
||||
cancel := d.cancelCh
|
||||
d.cancelLock.RUnlock()
|
||||
|
||||
select {
|
||||
case d.bodyCh <- bodyPack{id, transactions, uncles}:
|
||||
return nil
|
||||
|
||||
case <-cancel:
|
||||
|
@ -27,20 +27,39 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/event"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
)
|
||||
|
||||
var (
|
||||
testdb, _ = ethdb.NewMemDatabase()
|
||||
genesis = core.GenesisBlockForTesting(testdb, common.Address{}, big.NewInt(0))
|
||||
testdb, _ = ethdb.NewMemDatabase()
|
||||
testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
|
||||
testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
|
||||
genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000))
|
||||
)
|
||||
|
||||
// makeChain creates a chain of n blocks starting at but not including
|
||||
// parent. the returned hash chain is ordered head->parent.
|
||||
// makeChain creates a chain of n blocks starting at and including parent.
|
||||
// the returned hash chain is ordered head->parent. In addition, every 3rd block
|
||||
// contains a transaction and every 5th an uncle to allow testing correct block
|
||||
// reassembly.
|
||||
func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
|
||||
blocks := core.GenerateChain(parent, testdb, n, func(i int, gen *core.BlockGen) {
|
||||
gen.SetCoinbase(common.Address{seed})
|
||||
blocks := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
|
||||
block.SetCoinbase(common.Address{seed})
|
||||
|
||||
// If the block number is multiple of 3, send a bonus transaction to the miner
|
||||
if parent == genesis && i%3 == 0 {
|
||||
tx, err := types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(testKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
block.AddTx(tx)
|
||||
}
|
||||
// If the block number is a multiple of 5, add a bonus uncle to the block
|
||||
if i%5 == 0 {
|
||||
block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))})
|
||||
}
|
||||
})
|
||||
hashes := make([]common.Hash, n+1)
|
||||
hashes[len(hashes)-1] = parent.Hash()
|
||||
@ -78,8 +97,6 @@ type downloadTester struct {
|
||||
ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester
|
||||
peerHashes map[string][]common.Hash // Hash chain belonging to different test peers
|
||||
peerBlocks map[string]map[common.Hash]*types.Block // Blocks belonging to different test peers
|
||||
|
||||
maxHashFetch int // Overrides the maximum number of retrieved hashes
|
||||
}
|
||||
|
||||
// newTester creates a new downloader test mocker.
|
||||
@ -156,7 +173,9 @@ func (dl *downloadTester) newPeer(id string, version int, hashes []common.Hash,
|
||||
// specific delay time on processing the network packets sent to it, simulating
|
||||
// potentially slow network IO.
|
||||
func (dl *downloadTester) newSlowPeer(id string, version int, hashes []common.Hash, blocks map[common.Hash]*types.Block, delay time.Duration) error {
|
||||
err := dl.downloader.RegisterPeer(id, version, hashes[0], dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, version, delay), dl.peerGetBlocksFn(id, delay))
|
||||
err := dl.downloader.RegisterPeer(id, version, hashes[0],
|
||||
dl.peerGetRelHashesFn(id, delay), dl.peerGetAbsHashesFn(id, delay), dl.peerGetBlocksFn(id, delay),
|
||||
nil, dl.peerGetAbsHeadersFn(id, delay), dl.peerGetBodiesFn(id, delay))
|
||||
if err == nil {
|
||||
// Assign the owned hashes and blocks to the peer (deep copy)
|
||||
dl.peerHashes[id] = make([]common.Hash, len(hashes))
|
||||
@ -184,13 +203,9 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun
|
||||
return func(head common.Hash) error {
|
||||
time.Sleep(delay)
|
||||
|
||||
limit := MaxHashFetch
|
||||
if dl.maxHashFetch > 0 {
|
||||
limit = dl.maxHashFetch
|
||||
}
|
||||
// Gather the next batch of hashes
|
||||
hashes := dl.peerHashes[id]
|
||||
result := make([]common.Hash, 0, limit)
|
||||
result := make([]common.Hash, 0, MaxHashFetch)
|
||||
for i, hash := range hashes {
|
||||
if hash == head {
|
||||
i++
|
||||
@ -204,7 +219,7 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun
|
||||
// Delay delivery a bit to allow attacks to unfold
|
||||
go func() {
|
||||
time.Sleep(time.Millisecond)
|
||||
dl.downloader.DeliverHashes(id, result)
|
||||
dl.downloader.DeliverHashes61(id, result)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
@ -213,24 +228,20 @@ func (dl *downloadTester) peerGetRelHashesFn(id string, delay time.Duration) fun
|
||||
// peerGetAbsHashesFn constructs a GetHashesFromNumber function associated with
|
||||
// a particular peer in the download tester. The returned function can be used to
|
||||
// retrieve batches of hashes from the particularly requested peer.
|
||||
func (dl *downloadTester) peerGetAbsHashesFn(id string, version int, delay time.Duration) func(uint64, int) error {
|
||||
func (dl *downloadTester) peerGetAbsHashesFn(id string, delay time.Duration) func(uint64, int) error {
|
||||
return func(head uint64, count int) error {
|
||||
time.Sleep(delay)
|
||||
|
||||
limit := count
|
||||
if dl.maxHashFetch > 0 {
|
||||
limit = dl.maxHashFetch
|
||||
}
|
||||
// Gather the next batch of hashes
|
||||
hashes := dl.peerHashes[id]
|
||||
result := make([]common.Hash, 0, limit)
|
||||
for i := 0; i < limit && len(hashes)-int(head)-1-i >= 0; i++ {
|
||||
result := make([]common.Hash, 0, count)
|
||||
for i := 0; i < count && len(hashes)-int(head)-1-i >= 0; i++ {
|
||||
result = append(result, hashes[len(hashes)-int(head)-1-i])
|
||||
}
|
||||
// Delay delivery a bit to allow attacks to unfold
|
||||
go func() {
|
||||
time.Sleep(time.Millisecond)
|
||||
dl.downloader.DeliverHashes(id, result)
|
||||
dl.downloader.DeliverHashes61(id, result)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
@ -249,7 +260,55 @@ func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([
|
||||
result = append(result, block)
|
||||
}
|
||||
}
|
||||
go dl.downloader.DeliverBlocks(id, result)
|
||||
go dl.downloader.DeliverBlocks61(id, result)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// peerGetAbsHeadersFn constructs a GetBlockHeaders function based on a numbered
|
||||
// origin; associated with a particular peer in the download tester. The returned
|
||||
// function can be used to retrieve batches of headers from the particular peer.
|
||||
func (dl *downloadTester) peerGetAbsHeadersFn(id string, delay time.Duration) func(uint64, int, int, bool) error {
|
||||
return func(origin uint64, amount int, skip int, reverse bool) error {
|
||||
time.Sleep(delay)
|
||||
|
||||
// Gather the next batch of hashes
|
||||
hashes := dl.peerHashes[id]
|
||||
blocks := dl.peerBlocks[id]
|
||||
result := make([]*types.Header, 0, amount)
|
||||
for i := 0; i < amount && len(hashes)-int(origin)-1-i >= 0; i++ {
|
||||
if block, ok := blocks[hashes[len(hashes)-int(origin)-1-i]]; ok {
|
||||
result = append(result, block.Header())
|
||||
}
|
||||
}
|
||||
// Delay delivery a bit to allow attacks to unfold
|
||||
go func() {
|
||||
time.Sleep(time.Millisecond)
|
||||
dl.downloader.DeliverHeaders(id, result)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// peerGetBodiesFn constructs a getBlockBodies method associated with a particular
|
||||
// peer in the download tester. The returned function can be used to retrieve
|
||||
// batches of block bodies from the particularly requested peer.
|
||||
func (dl *downloadTester) peerGetBodiesFn(id string, delay time.Duration) func([]common.Hash) error {
|
||||
return func(hashes []common.Hash) error {
|
||||
time.Sleep(delay)
|
||||
blocks := dl.peerBlocks[id]
|
||||
|
||||
transactions := make([][]*types.Transaction, 0, len(hashes))
|
||||
uncles := make([][]*types.Header, 0, len(hashes))
|
||||
|
||||
for _, hash := range hashes {
|
||||
if block, ok := blocks[hash]; ok {
|
||||
transactions = append(transactions, block.Transactions())
|
||||
uncles = append(uncles, block.Uncles())
|
||||
}
|
||||
}
|
||||
go dl.downloader.DeliverBodies(id, transactions, uncles)
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -258,13 +317,18 @@ func (dl *downloadTester) peerGetBlocksFn(id string, delay time.Duration) func([
|
||||
// Tests that simple synchronization against a canonical chain works correctly.
|
||||
// In this test common ancestor lookup should be short circuited and not require
|
||||
// binary searching.
|
||||
func TestCanonicalSynchronisation61(t *testing.T) {
|
||||
func TestCanonicalSynchronisation61(t *testing.T) { testCanonicalSynchronisation(t, 61) }
|
||||
func TestCanonicalSynchronisation62(t *testing.T) { testCanonicalSynchronisation(t, 62) }
|
||||
func TestCanonicalSynchronisation63(t *testing.T) { testCanonicalSynchronisation(t, 63) }
|
||||
func TestCanonicalSynchronisation64(t *testing.T) { testCanonicalSynchronisation(t, 64) }
|
||||
|
||||
func testCanonicalSynchronisation(t *testing.T, protocol int) {
|
||||
// Create a small enough block chain to download
|
||||
targetBlocks := blockCacheLimit - 15
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
tester.newPeer("peer", eth61, hashes, blocks)
|
||||
tester.newPeer("peer", protocol, hashes, blocks)
|
||||
|
||||
// Synchronise with the peer and make sure all blocks were retrieved
|
||||
if err := tester.sync("peer", nil); err != nil {
|
||||
@ -277,7 +341,10 @@ func TestCanonicalSynchronisation61(t *testing.T) {
|
||||
|
||||
// Tests that if a large batch of blocks are being downloaded, it is throttled
|
||||
// until the cached blocks are retrieved.
|
||||
func TestThrottling61(t *testing.T) { testThrottling(t, eth61) }
|
||||
func TestThrottling61(t *testing.T) { testThrottling(t, 61) }
|
||||
func TestThrottling62(t *testing.T) { testThrottling(t, 62) }
|
||||
func TestThrottling63(t *testing.T) { testThrottling(t, 63) }
|
||||
func TestThrottling64(t *testing.T) { testThrottling(t, 64) }
|
||||
|
||||
func testThrottling(t *testing.T, protocol int) {
|
||||
// Create a long block chain to download and the tester
|
||||
@ -288,11 +355,10 @@ func testThrottling(t *testing.T, protocol int) {
|
||||
tester.newPeer("peer", protocol, hashes, blocks)
|
||||
|
||||
// Wrap the importer to allow stepping
|
||||
done := make(chan int)
|
||||
tester.downloader.insertChain = func(blocks types.Blocks) (int, error) {
|
||||
n, err := tester.insertChain(blocks)
|
||||
done <- n
|
||||
return n, err
|
||||
blocked, proceed := uint32(0), make(chan struct{})
|
||||
tester.downloader.chainInsertHook = func(blocks []*Block) {
|
||||
atomic.StoreUint32(&blocked, uint32(len(blocks)))
|
||||
<-proceed
|
||||
}
|
||||
// Start a synchronisation concurrently
|
||||
errc := make(chan error)
|
||||
@ -303,27 +369,25 @@ func testThrottling(t *testing.T, protocol int) {
|
||||
for len(tester.ownBlocks) < targetBlocks+1 {
|
||||
// Wait a bit for sync to throttle itself
|
||||
var cached int
|
||||
for start := time.Now(); time.Since(start) < 3*time.Second; {
|
||||
for start := time.Now(); time.Since(start) < time.Second; {
|
||||
time.Sleep(25 * time.Millisecond)
|
||||
|
||||
cached = len(tester.downloader.queue.blockPool)
|
||||
if cached == blockCacheLimit || len(tester.ownBlocks)+cached == targetBlocks+1 {
|
||||
if cached == blockCacheLimit || len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) == targetBlocks+1 {
|
||||
break
|
||||
}
|
||||
}
|
||||
// Make sure we filled up the cache, then exhaust it
|
||||
time.Sleep(25 * time.Millisecond) // give it a chance to screw up
|
||||
if cached != blockCacheLimit && len(tester.ownBlocks)+cached < targetBlocks+1 {
|
||||
t.Fatalf("block count mismatch: have %v, want %v", cached, blockCacheLimit)
|
||||
if cached != blockCacheLimit && len(tester.ownBlocks)+cached+int(atomic.LoadUint32(&blocked)) != targetBlocks+1 {
|
||||
t.Fatalf("block count mismatch: have %v, want %v (owned %v, target %v)", cached, blockCacheLimit, len(tester.ownBlocks), targetBlocks+1)
|
||||
}
|
||||
<-done // finish previous blocking import
|
||||
for cached > maxBlockProcess {
|
||||
cached -= <-done
|
||||
// Permit the blocked blocks to import
|
||||
if atomic.LoadUint32(&blocked) > 0 {
|
||||
atomic.StoreUint32(&blocked, uint32(0))
|
||||
proceed <- struct{}{}
|
||||
}
|
||||
time.Sleep(25 * time.Millisecond) // yield to the insertion
|
||||
}
|
||||
<-done // finish the last blocking import
|
||||
|
||||
// Check that we haven't pulled more blocks than available
|
||||
if len(tester.ownBlocks) > targetBlocks+1 {
|
||||
t.Fatalf("target block count mismatch: have %v, want %v", len(tester.ownBlocks), targetBlocks+1)
|
||||
@ -336,14 +400,19 @@ func testThrottling(t *testing.T, protocol int) {
|
||||
// Tests that simple synchronization against a forked chain works correctly. In
|
||||
// this test common ancestor lookup should *not* be short circuited, and a full
|
||||
// binary search should be executed.
|
||||
func TestForkedSynchronisation61(t *testing.T) {
|
||||
func TestForkedSynchronisation61(t *testing.T) { testForkedSynchronisation(t, 61) }
|
||||
func TestForkedSynchronisation62(t *testing.T) { testForkedSynchronisation(t, 62) }
|
||||
func TestForkedSynchronisation63(t *testing.T) { testForkedSynchronisation(t, 63) }
|
||||
func TestForkedSynchronisation64(t *testing.T) { testForkedSynchronisation(t, 64) }
|
||||
|
||||
func testForkedSynchronisation(t *testing.T, protocol int) {
|
||||
// Create a long enough forked chain
|
||||
common, fork := MaxHashFetch, 2*MaxHashFetch
|
||||
hashesA, hashesB, blocksA, blocksB := makeChainFork(common+fork, fork, genesis)
|
||||
|
||||
tester := newTester()
|
||||
tester.newPeer("fork A", eth61, hashesA, blocksA)
|
||||
tester.newPeer("fork B", eth61, hashesB, blocksB)
|
||||
tester.newPeer("fork A", protocol, hashesA, blocksA)
|
||||
tester.newPeer("fork B", protocol, hashesB, blocksB)
|
||||
|
||||
// Synchronise with the peer and make sure all blocks were retrieved
|
||||
if err := tester.sync("fork A", nil); err != nil {
|
||||
@ -362,20 +431,36 @@ func TestForkedSynchronisation61(t *testing.T) {
|
||||
}
|
||||
|
||||
// Tests that an inactive downloader will not accept incoming hashes and blocks.
|
||||
func TestInactiveDownloader(t *testing.T) {
|
||||
func TestInactiveDownloader61(t *testing.T) {
|
||||
tester := newTester()
|
||||
|
||||
// Check that neither hashes nor blocks are accepted
|
||||
if err := tester.downloader.DeliverHashes("bad peer", []common.Hash{}); err != errNoSyncActive {
|
||||
if err := tester.downloader.DeliverHashes61("bad peer", []common.Hash{}); err != errNoSyncActive {
|
||||
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
|
||||
}
|
||||
if err := tester.downloader.DeliverBlocks("bad peer", []*types.Block{}); err != errNoSyncActive {
|
||||
if err := tester.downloader.DeliverBlocks61("bad peer", []*types.Block{}); err != errNoSyncActive {
|
||||
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that an inactive downloader will not accept incoming block headers and bodies.
|
||||
func TestInactiveDownloader62(t *testing.T) {
|
||||
tester := newTester()
|
||||
|
||||
// Check that neither block headers nor bodies are accepted
|
||||
if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive {
|
||||
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
|
||||
}
|
||||
if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive {
|
||||
t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that a canceled download wipes all previously accumulated state.
|
||||
func TestCancel61(t *testing.T) { testCancel(t, eth61) }
|
||||
func TestCancel61(t *testing.T) { testCancel(t, 61) }
|
||||
func TestCancel62(t *testing.T) { testCancel(t, 62) }
|
||||
func TestCancel63(t *testing.T) { testCancel(t, 63) }
|
||||
func TestCancel64(t *testing.T) { testCancel(t, 64) }
|
||||
|
||||
func testCancel(t *testing.T, protocol int) {
|
||||
// Create a small enough block chain to download and the tester
|
||||
@ -383,6 +468,9 @@ func testCancel(t *testing.T, protocol int) {
|
||||
if targetBlocks >= MaxHashFetch {
|
||||
targetBlocks = MaxHashFetch - 15
|
||||
}
|
||||
if targetBlocks >= MaxHeaderFetch {
|
||||
targetBlocks = MaxHeaderFetch - 15
|
||||
}
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
@ -390,27 +478,30 @@ func testCancel(t *testing.T, protocol int) {
|
||||
|
||||
// Make sure canceling works with a pristine downloader
|
||||
tester.downloader.cancel()
|
||||
hashCount, blockCount := tester.downloader.queue.Size()
|
||||
if hashCount > 0 || blockCount > 0 {
|
||||
t.Errorf("block or hash count mismatch: %d hashes, %d blocks, want 0", hashCount, blockCount)
|
||||
downloading, importing := tester.downloader.queue.Size()
|
||||
if downloading > 0 || importing > 0 {
|
||||
t.Errorf("download or import count mismatch: %d downloading, %d importing, want 0", downloading, importing)
|
||||
}
|
||||
// Synchronise with the peer, but cancel afterwards
|
||||
if err := tester.sync("peer", nil); err != nil {
|
||||
t.Fatalf("failed to synchronise blocks: %v", err)
|
||||
}
|
||||
tester.downloader.cancel()
|
||||
hashCount, blockCount = tester.downloader.queue.Size()
|
||||
if hashCount > 0 || blockCount > 0 {
|
||||
t.Errorf("block or hash count mismatch: %d hashes, %d blocks, want 0", hashCount, blockCount)
|
||||
downloading, importing = tester.downloader.queue.Size()
|
||||
if downloading > 0 || importing > 0 {
|
||||
t.Errorf("download or import count mismatch: %d downloading, %d importing, want 0", downloading, importing)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that synchronisation from multiple peers works as intended (multi thread sanity test).
|
||||
func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, eth61) }
|
||||
func TestMultiSynchronisation61(t *testing.T) { testMultiSynchronisation(t, 61) }
|
||||
func TestMultiSynchronisation62(t *testing.T) { testMultiSynchronisation(t, 62) }
|
||||
func TestMultiSynchronisation63(t *testing.T) { testMultiSynchronisation(t, 63) }
|
||||
func TestMultiSynchronisation64(t *testing.T) { testMultiSynchronisation(t, 64) }
|
||||
|
||||
func testMultiSynchronisation(t *testing.T, protocol int) {
|
||||
// Create various peers with various parts of the chain
|
||||
targetPeers := 16
|
||||
targetPeers := 8
|
||||
targetBlocks := targetPeers*blockCacheLimit - 15
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
@ -436,45 +527,130 @@ func testMultiSynchronisation(t *testing.T, protocol int) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that if a block is empty (i.e. header only), no body request should be
|
||||
// made, and instead the header should be assembled into a whole block in itself.
|
||||
func TestEmptyBlockShortCircuit62(t *testing.T) { testEmptyBlockShortCircuit(t, 62) }
|
||||
func TestEmptyBlockShortCircuit63(t *testing.T) { testEmptyBlockShortCircuit(t, 63) }
|
||||
func TestEmptyBlockShortCircuit64(t *testing.T) { testEmptyBlockShortCircuit(t, 64) }
|
||||
|
||||
func testEmptyBlockShortCircuit(t *testing.T, protocol int) {
|
||||
// Create a small enough block chain to download
|
||||
targetBlocks := blockCacheLimit - 15
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
tester.newPeer("peer", protocol, hashes, blocks)
|
||||
|
||||
// Instrument the downloader to signal body requests
|
||||
requested := int32(0)
|
||||
tester.downloader.bodyFetchHook = func(headers []*types.Header) {
|
||||
atomic.AddInt32(&requested, int32(len(headers)))
|
||||
}
|
||||
// Synchronise with the peer and make sure all blocks were retrieved
|
||||
if err := tester.sync("peer", nil); err != nil {
|
||||
t.Fatalf("failed to synchronise blocks: %v", err)
|
||||
}
|
||||
if imported := len(tester.ownBlocks); imported != targetBlocks+1 {
|
||||
t.Fatalf("synchronised block mismatch: have %v, want %v", imported, targetBlocks+1)
|
||||
}
|
||||
// Validate the number of block bodies that should have been requested
|
||||
needed := 0
|
||||
for _, block := range blocks {
|
||||
if block != genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) {
|
||||
needed++
|
||||
}
|
||||
}
|
||||
if int(requested) != needed {
|
||||
t.Fatalf("block body retrieval count mismatch: have %v, want %v", requested, needed)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that if a peer sends an invalid body for a requested block, it gets
|
||||
// dropped immediately by the downloader.
|
||||
func TestInvalidBlockBodyAttack62(t *testing.T) { testInvalidBlockBodyAttack(t, 62) }
|
||||
func TestInvalidBlockBodyAttack63(t *testing.T) { testInvalidBlockBodyAttack(t, 63) }
|
||||
func TestInvalidBlockBodyAttack64(t *testing.T) { testInvalidBlockBodyAttack(t, 64) }
|
||||
|
||||
func testInvalidBlockBodyAttack(t *testing.T, protocol int) {
|
||||
// Create two peers, one feeding invalid block bodies
|
||||
targetBlocks := 4*blockCacheLimit - 15
|
||||
hashes, validBlocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
invalidBlocks := make(map[common.Hash]*types.Block)
|
||||
for hash, block := range validBlocks {
|
||||
invalidBlocks[hash] = types.NewBlockWithHeader(block.Header())
|
||||
}
|
||||
|
||||
tester := newTester()
|
||||
tester.newPeer("valid", protocol, hashes, validBlocks)
|
||||
tester.newPeer("attack", protocol, hashes, invalidBlocks)
|
||||
|
||||
// Synchronise with the valid peer (will pull contents from the attacker too)
|
||||
if err := tester.sync("valid", nil); err != nil {
|
||||
t.Fatalf("failed to synchronise blocks: %v", err)
|
||||
}
|
||||
if imported := len(tester.ownBlocks); imported != len(hashes) {
|
||||
t.Fatalf("synchronised block mismatch: have %v, want %v", imported, len(hashes))
|
||||
}
|
||||
// Make sure the attacker was detected and dropped in the mean time
|
||||
if _, ok := tester.peerHashes["attack"]; ok {
|
||||
t.Fatalf("block body attacker not detected/dropped")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that a peer advertising an high TD doesn't get to stall the downloader
|
||||
// afterwards by not sending any useful hashes.
|
||||
func TestHighTDStarvationAttack61(t *testing.T) {
|
||||
func TestHighTDStarvationAttack61(t *testing.T) { testHighTDStarvationAttack(t, 61) }
|
||||
func TestHighTDStarvationAttack62(t *testing.T) { testHighTDStarvationAttack(t, 62) }
|
||||
func TestHighTDStarvationAttack63(t *testing.T) { testHighTDStarvationAttack(t, 63) }
|
||||
func TestHighTDStarvationAttack64(t *testing.T) { testHighTDStarvationAttack(t, 64) }
|
||||
|
||||
func testHighTDStarvationAttack(t *testing.T, protocol int) {
|
||||
tester := newTester()
|
||||
tester.newPeer("attack", eth61, []common.Hash{genesis.Hash()}, nil)
|
||||
hashes, blocks := makeChain(0, 0, genesis)
|
||||
|
||||
tester.newPeer("attack", protocol, []common.Hash{hashes[0]}, blocks)
|
||||
if err := tester.sync("attack", big.NewInt(1000000)); err != errStallingPeer {
|
||||
t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer)
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that misbehaving peers are disconnected, whilst behaving ones are not.
|
||||
func TestHashAttackerDropping(t *testing.T) {
|
||||
func TestBlockHeaderAttackerDropping61(t *testing.T) { testBlockHeaderAttackerDropping(t, 61) }
|
||||
func TestBlockHeaderAttackerDropping62(t *testing.T) { testBlockHeaderAttackerDropping(t, 62) }
|
||||
func TestBlockHeaderAttackerDropping63(t *testing.T) { testBlockHeaderAttackerDropping(t, 63) }
|
||||
func TestBlockHeaderAttackerDropping64(t *testing.T) { testBlockHeaderAttackerDropping(t, 64) }
|
||||
|
||||
func testBlockHeaderAttackerDropping(t *testing.T, protocol int) {
|
||||
// Define the disconnection requirement for individual hash fetch errors
|
||||
tests := []struct {
|
||||
result error
|
||||
drop bool
|
||||
}{
|
||||
{nil, false}, // Sync succeeded, all is well
|
||||
{errBusy, false}, // Sync is already in progress, no problem
|
||||
{errUnknownPeer, false}, // Peer is unknown, was already dropped, don't double drop
|
||||
{errBadPeer, true}, // Peer was deemed bad for some reason, drop it
|
||||
{errStallingPeer, true}, // Peer was detected to be stalling, drop it
|
||||
{errBannedHead, true}, // Peer's head hash is a known bad hash, drop it
|
||||
{errNoPeers, false}, // No peers to download from, soft race, no issue
|
||||
{errPendingQueue, false}, // There are blocks still cached, wait to exhaust, no issue
|
||||
{errTimeout, true}, // No hashes received in due time, drop the peer
|
||||
{errEmptyHashSet, true}, // No hashes were returned as a response, drop as it's a dead end
|
||||
{errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser
|
||||
{errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop
|
||||
{errCrossCheckFailed, true}, // Hash-origin failed to pass a block cross check, drop
|
||||
{errCancelHashFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
{errCancelBlockFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
{nil, false}, // Sync succeeded, all is well
|
||||
{errBusy, false}, // Sync is already in progress, no problem
|
||||
{errUnknownPeer, false}, // Peer is unknown, was already dropped, don't double drop
|
||||
{errBadPeer, true}, // Peer was deemed bad for some reason, drop it
|
||||
{errStallingPeer, true}, // Peer was detected to be stalling, drop it
|
||||
{errNoPeers, false}, // No peers to download from, soft race, no issue
|
||||
{errPendingQueue, false}, // There are blocks still cached, wait to exhaust, no issue
|
||||
{errTimeout, true}, // No hashes received in due time, drop the peer
|
||||
{errEmptyHashSet, true}, // No hashes were returned as a response, drop as it's a dead end
|
||||
{errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end
|
||||
{errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser
|
||||
{errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop
|
||||
{errInvalidBody, false}, // A bad peer was detected, but not the sync origin
|
||||
{errCancelHashFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
{errCancelBlockFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
{errCancelHeaderFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
{errCancelBodyFetch, false}, // Synchronisation was canceled, origin may be innocent, don't drop
|
||||
}
|
||||
// Run the tests and check disconnection status
|
||||
tester := newTester()
|
||||
for i, tt := range tests {
|
||||
// Register a new peer and ensure it's presence
|
||||
id := fmt.Sprintf("test %d", i)
|
||||
if err := tester.newPeer(id, eth61, []common.Hash{genesis.Hash()}, nil); err != nil {
|
||||
if err := tester.newPeer(id, protocol, []common.Hash{genesis.Hash()}, nil); err != nil {
|
||||
t.Fatalf("test %d: failed to register new peer: %v", i, err)
|
||||
}
|
||||
if _, ok := tester.peerHashes[id]; !ok {
|
||||
@ -491,7 +667,12 @@ func TestHashAttackerDropping(t *testing.T) {
|
||||
}
|
||||
|
||||
// Tests that feeding bad blocks will result in a peer drop.
|
||||
func TestBlockAttackerDropping(t *testing.T) {
|
||||
func TestBlockBodyAttackerDropping61(t *testing.T) { testBlockBodyAttackerDropping(t, 61) }
|
||||
func TestBlockBodyAttackerDropping62(t *testing.T) { testBlockBodyAttackerDropping(t, 62) }
|
||||
func TestBlockBodyAttackerDropping63(t *testing.T) { testBlockBodyAttackerDropping(t, 63) }
|
||||
func TestBlockBodyAttackerDropping64(t *testing.T) { testBlockBodyAttackerDropping(t, 64) }
|
||||
|
||||
func testBlockBodyAttackerDropping(t *testing.T, protocol int) {
|
||||
// Define the disconnection requirement for individual block import errors
|
||||
tests := []struct {
|
||||
failure bool
|
||||
@ -506,7 +687,7 @@ func TestBlockAttackerDropping(t *testing.T) {
|
||||
for i, tt := range tests {
|
||||
// Register a new peer and ensure it's presence
|
||||
id := fmt.Sprintf("test %d", i)
|
||||
if err := tester.newPeer(id, eth61, []common.Hash{common.Hash{}}, nil); err != nil {
|
||||
if err := tester.newPeer(id, protocol, []common.Hash{common.Hash{}}, nil); err != nil {
|
||||
t.Fatalf("test %d: failed to register new peer: %v", i, err)
|
||||
}
|
||||
if _, ok := tester.peerHashes[id]; !ok {
|
||||
|
@ -31,10 +31,16 @@ import (
|
||||
"gopkg.in/fatih/set.v0"
|
||||
)
|
||||
|
||||
// Hash and block fetchers belonging to eth/61 and below
|
||||
type relativeHashFetcherFn func(common.Hash) error
|
||||
type absoluteHashFetcherFn func(uint64, int) error
|
||||
type blockFetcherFn func([]common.Hash) error
|
||||
|
||||
// Block header and body fethers belonging to eth/62 and above
|
||||
type relativeHeaderFetcherFn func(common.Hash, int, int, bool) error
|
||||
type absoluteHeaderFetcherFn func(uint64, int, int, bool) error
|
||||
type blockBodyFetcherFn func([]common.Hash) error
|
||||
|
||||
var (
|
||||
errAlreadyFetching = errors.New("already fetching blocks from peer")
|
||||
errAlreadyRegistered = errors.New("peer is already registered")
|
||||
@ -54,25 +60,37 @@ type peer struct {
|
||||
|
||||
ignored *set.Set // Set of hashes not to request (didn't have previously)
|
||||
|
||||
getRelHashes relativeHashFetcherFn // Method to retrieve a batch of hashes from an origin hash
|
||||
getAbsHashes absoluteHashFetcherFn // Method to retrieve a batch of hashes from an absolute position
|
||||
getBlocks blockFetcherFn // Method to retrieve a batch of blocks
|
||||
getRelHashes relativeHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an origin hash
|
||||
getAbsHashes absoluteHashFetcherFn // [eth/61] Method to retrieve a batch of hashes from an absolute position
|
||||
getBlocks blockFetcherFn // [eth/61] Method to retrieve a batch of blocks
|
||||
|
||||
getRelHeaders relativeHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an origin hash
|
||||
getAbsHeaders absoluteHeaderFetcherFn // [eth/62] Method to retrieve a batch of headers from an absolute position
|
||||
getBlockBodies blockBodyFetcherFn // [eth/62] Method to retrieve a batch of block bodies
|
||||
|
||||
version int // Eth protocol version number to switch strategies
|
||||
}
|
||||
|
||||
// newPeer create a new downloader peer, with specific hash and block retrieval
|
||||
// mechanisms.
|
||||
func newPeer(id string, version int, head common.Hash, getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn) *peer {
|
||||
func newPeer(id string, version int, head common.Hash,
|
||||
getRelHashes relativeHashFetcherFn, getAbsHashes absoluteHashFetcherFn, getBlocks blockFetcherFn, // eth/61 callbacks, remove when upgrading
|
||||
getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn) *peer {
|
||||
return &peer{
|
||||
id: id,
|
||||
head: head,
|
||||
capacity: 1,
|
||||
id: id,
|
||||
head: head,
|
||||
capacity: 1,
|
||||
ignored: set.New(),
|
||||
|
||||
getRelHashes: getRelHashes,
|
||||
getAbsHashes: getAbsHashes,
|
||||
getBlocks: getBlocks,
|
||||
ignored: set.New(),
|
||||
version: version,
|
||||
|
||||
getRelHeaders: getRelHeaders,
|
||||
getAbsHeaders: getAbsHeaders,
|
||||
getBlockBodies: getBlockBodies,
|
||||
|
||||
version: version,
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,8 +101,8 @@ func (p *peer) Reset() {
|
||||
p.ignored.Clear()
|
||||
}
|
||||
|
||||
// Fetch sends a block retrieval request to the remote peer.
|
||||
func (p *peer) Fetch(request *fetchRequest) error {
|
||||
// Fetch61 sends a block retrieval request to the remote peer.
|
||||
func (p *peer) Fetch61(request *fetchRequest) error {
|
||||
// Short circuit if the peer is already fetching
|
||||
if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
|
||||
return errAlreadyFetching
|
||||
@ -101,10 +119,28 @@ func (p *peer) Fetch(request *fetchRequest) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetIdle sets the peer to idle, allowing it to execute new retrieval requests.
|
||||
// Fetch sends a block body retrieval request to the remote peer.
|
||||
func (p *peer) Fetch(request *fetchRequest) error {
|
||||
// Short circuit if the peer is already fetching
|
||||
if !atomic.CompareAndSwapInt32(&p.idle, 0, 1) {
|
||||
return errAlreadyFetching
|
||||
}
|
||||
p.started = time.Now()
|
||||
|
||||
// Convert the header set to a retrievable slice
|
||||
hashes := make([]common.Hash, 0, len(request.Headers))
|
||||
for _, header := range request.Headers {
|
||||
hashes = append(hashes, header.Hash())
|
||||
}
|
||||
go p.getBlockBodies(hashes)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetIdle61 sets the peer to idle, allowing it to execute new retrieval requests.
|
||||
// Its block retrieval allowance will also be updated either up- or downwards,
|
||||
// depending on whether the previous fetch completed in time or not.
|
||||
func (p *peer) SetIdle() {
|
||||
func (p *peer) SetIdle61() {
|
||||
// Update the peer's download allowance based on previous performance
|
||||
scale := 2.0
|
||||
if time.Since(p.started) > blockSoftTTL {
|
||||
@ -131,6 +167,36 @@ func (p *peer) SetIdle() {
|
||||
atomic.StoreInt32(&p.idle, 0)
|
||||
}
|
||||
|
||||
// SetIdle sets the peer to idle, allowing it to execute new retrieval requests.
|
||||
// Its block body retrieval allowance will also be updated either up- or downwards,
|
||||
// depending on whether the previous fetch completed in time or not.
|
||||
func (p *peer) SetIdle() {
|
||||
// Update the peer's download allowance based on previous performance
|
||||
scale := 2.0
|
||||
if time.Since(p.started) > bodySoftTTL {
|
||||
scale = 0.5
|
||||
if time.Since(p.started) > bodyHardTTL {
|
||||
scale = 1 / float64(MaxBodyFetch) // reduces capacity to 1
|
||||
}
|
||||
}
|
||||
for {
|
||||
// Calculate the new download bandwidth allowance
|
||||
prev := atomic.LoadInt32(&p.capacity)
|
||||
next := int32(math.Max(1, math.Min(float64(MaxBodyFetch), float64(prev)*scale)))
|
||||
|
||||
// Try to update the old value
|
||||
if atomic.CompareAndSwapInt32(&p.capacity, prev, next) {
|
||||
// If we're having problems at 1 capacity, try to find better peers
|
||||
if next == 1 {
|
||||
p.Demote()
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
// Set the peer to idle to allow further block requests
|
||||
atomic.StoreInt32(&p.idle, 0)
|
||||
}
|
||||
|
||||
// Capacity retrieves the peers block download allowance based on its previously
|
||||
// discovered bandwidth capacity.
|
||||
func (p *peer) Capacity() int {
|
||||
|
@ -43,16 +43,20 @@ var (
|
||||
|
||||
// fetchRequest is a currently running block retrieval operation.
|
||||
type fetchRequest struct {
|
||||
Peer *peer // Peer to which the request was sent
|
||||
Hashes map[common.Hash]int // Requested hashes with their insertion index (priority)
|
||||
Time time.Time // Time when the request was made
|
||||
Peer *peer // Peer to which the request was sent
|
||||
Hashes map[common.Hash]int // [eth/61] Requested hashes with their insertion index (priority)
|
||||
Headers []*types.Header // [eth/62] Requested headers, sorted by request order
|
||||
Time time.Time // Time when the request was made
|
||||
}
|
||||
|
||||
// queue represents hashes that are either need fetching or are being fetched
|
||||
type queue struct {
|
||||
hashPool map[common.Hash]int // Pending hashes, mapping to their insertion index (priority)
|
||||
hashQueue *prque.Prque // Priority queue of the block hashes to fetch
|
||||
hashCounter int // Counter indexing the added hashes to ensure retrieval order
|
||||
hashPool map[common.Hash]int // [eth/61] Pending hashes, mapping to their insertion index (priority)
|
||||
hashQueue *prque.Prque // [eth/61] Priority queue of the block hashes to fetch
|
||||
hashCounter int // [eth/61] Counter indexing the added hashes to ensure retrieval order
|
||||
|
||||
headerPool map[common.Hash]*types.Header // [eth/62] Pending headers, mapping from their hashes
|
||||
headerQueue *prque.Prque // [eth/62] Priority queue of the headers to fetch the bodies for
|
||||
|
||||
pendPool map[string]*fetchRequest // Currently pending block retrieval operations
|
||||
|
||||
@ -66,11 +70,13 @@ type queue struct {
|
||||
// newQueue creates a new download queue for scheduling block retrieval.
|
||||
func newQueue() *queue {
|
||||
return &queue{
|
||||
hashPool: make(map[common.Hash]int),
|
||||
hashQueue: prque.New(),
|
||||
pendPool: make(map[string]*fetchRequest),
|
||||
blockPool: make(map[common.Hash]uint64),
|
||||
blockCache: make([]*Block, blockCacheLimit),
|
||||
hashPool: make(map[common.Hash]int),
|
||||
hashQueue: prque.New(),
|
||||
headerPool: make(map[common.Hash]*types.Header),
|
||||
headerQueue: prque.New(),
|
||||
pendPool: make(map[string]*fetchRequest),
|
||||
blockPool: make(map[common.Hash]uint64),
|
||||
blockCache: make([]*Block, blockCacheLimit),
|
||||
}
|
||||
}
|
||||
|
||||
@ -83,6 +89,9 @@ func (q *queue) Reset() {
|
||||
q.hashQueue.Reset()
|
||||
q.hashCounter = 0
|
||||
|
||||
q.headerPool = make(map[common.Hash]*types.Header)
|
||||
q.headerQueue.Reset()
|
||||
|
||||
q.pendPool = make(map[string]*fetchRequest)
|
||||
|
||||
q.blockPool = make(map[common.Hash]uint64)
|
||||
@ -90,21 +99,21 @@ func (q *queue) Reset() {
|
||||
q.blockCache = make([]*Block, blockCacheLimit)
|
||||
}
|
||||
|
||||
// Size retrieves the number of hashes in the queue, returning separately for
|
||||
// Size retrieves the number of blocks in the queue, returning separately for
|
||||
// pending and already downloaded.
|
||||
func (q *queue) Size() (int, int) {
|
||||
q.lock.RLock()
|
||||
defer q.lock.RUnlock()
|
||||
|
||||
return len(q.hashPool), len(q.blockPool)
|
||||
return len(q.hashPool) + len(q.headerPool), len(q.blockPool)
|
||||
}
|
||||
|
||||
// Pending retrieves the number of hashes pending for retrieval.
|
||||
// Pending retrieves the number of blocks pending for retrieval.
|
||||
func (q *queue) Pending() int {
|
||||
q.lock.RLock()
|
||||
defer q.lock.RUnlock()
|
||||
|
||||
return q.hashQueue.Size()
|
||||
return q.hashQueue.Size() + q.headerQueue.Size()
|
||||
}
|
||||
|
||||
// InFlight retrieves the number of fetch requests currently in flight.
|
||||
@ -124,7 +133,7 @@ func (q *queue) Throttle() bool {
|
||||
// Calculate the currently in-flight block requests
|
||||
pending := 0
|
||||
for _, request := range q.pendPool {
|
||||
pending += len(request.Hashes)
|
||||
pending += len(request.Hashes) + len(request.Headers)
|
||||
}
|
||||
// Throttle if more blocks are in-flight than free space in the cache
|
||||
return pending >= len(q.blockCache)-len(q.blockPool)
|
||||
@ -138,15 +147,18 @@ func (q *queue) Has(hash common.Hash) bool {
|
||||
if _, ok := q.hashPool[hash]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := q.headerPool[hash]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := q.blockPool[hash]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Insert adds a set of hashes for the download queue for scheduling, returning
|
||||
// Insert61 adds a set of hashes for the download queue for scheduling, returning
|
||||
// the new hashes encountered.
|
||||
func (q *queue) Insert(hashes []common.Hash, fifo bool) []common.Hash {
|
||||
func (q *queue) Insert61(hashes []common.Hash, fifo bool) []common.Hash {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
@ -172,6 +184,29 @@ func (q *queue) Insert(hashes []common.Hash, fifo bool) []common.Hash {
|
||||
return inserts
|
||||
}
|
||||
|
||||
// Insert adds a set of headers for the download queue for scheduling, returning
|
||||
// the new headers encountered.
|
||||
func (q *queue) Insert(headers []*types.Header) []*types.Header {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
// Insert all the headers prioritized by the contained block number
|
||||
inserts := make([]*types.Header, 0, len(headers))
|
||||
for _, header := range headers {
|
||||
// Make sure no duplicate requests are executed
|
||||
hash := header.Hash()
|
||||
if _, ok := q.headerPool[hash]; ok {
|
||||
glog.V(logger.Warn).Infof("Header %x already scheduled", hash)
|
||||
continue
|
||||
}
|
||||
// Queue the header for body retrieval
|
||||
inserts = append(inserts, header)
|
||||
q.headerPool[hash] = header
|
||||
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
|
||||
}
|
||||
return inserts
|
||||
}
|
||||
|
||||
// GetHeadBlock retrieves the first block from the cache, or nil if it hasn't
|
||||
// been downloaded yet (or simply non existent).
|
||||
func (q *queue) GetHeadBlock() *Block {
|
||||
@ -227,9 +262,9 @@ func (q *queue) TakeBlocks() []*Block {
|
||||
return blocks
|
||||
}
|
||||
|
||||
// Reserve reserves a set of hashes for the given peer, skipping any previously
|
||||
// Reserve61 reserves a set of hashes for the given peer, skipping any previously
|
||||
// failed download.
|
||||
func (q *queue) Reserve(p *peer, count int) *fetchRequest {
|
||||
func (q *queue) Reserve61(p *peer, count int) *fetchRequest {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
@ -276,6 +311,68 @@ func (q *queue) Reserve(p *peer, count int) *fetchRequest {
|
||||
return request
|
||||
}
|
||||
|
||||
// Reserve reserves a set of headers for the given peer, skipping any previously
|
||||
// failed download. Beside the next batch of needed fetches, it also returns a
|
||||
// flag whether empty blocks were queued requiring processing.
|
||||
func (q *queue) Reserve(p *peer, count int) (*fetchRequest, bool, error) {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
// Short circuit if the pool has been depleted, or if the peer's already
|
||||
// downloading something (sanity check not to corrupt state)
|
||||
if q.headerQueue.Empty() {
|
||||
return nil, false, nil
|
||||
}
|
||||
if _, ok := q.pendPool[p.id]; ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
// Calculate an upper limit on the bodies we might fetch (i.e. throttling)
|
||||
space := len(q.blockCache) - len(q.blockPool)
|
||||
for _, request := range q.pendPool {
|
||||
space -= len(request.Headers)
|
||||
}
|
||||
// Retrieve a batch of headers, skipping previously failed ones
|
||||
send := make([]*types.Header, 0, count)
|
||||
skip := make([]*types.Header, 0)
|
||||
|
||||
process := false
|
||||
for proc := 0; proc < space && len(send) < count && !q.headerQueue.Empty(); proc++ {
|
||||
header := q.headerQueue.PopItem().(*types.Header)
|
||||
|
||||
// If the header defines an empty block, deliver straight
|
||||
if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
|
||||
if err := q.enqueue("", types.NewBlockWithHeader(header)); err != nil {
|
||||
return nil, false, errInvalidChain
|
||||
}
|
||||
delete(q.headerPool, header.Hash())
|
||||
process, space, proc = true, space-1, proc-1
|
||||
continue
|
||||
}
|
||||
// If it's a content block, add to the body fetch request
|
||||
if p.ignored.Has(header.Hash()) {
|
||||
skip = append(skip, header)
|
||||
} else {
|
||||
send = append(send, header)
|
||||
}
|
||||
}
|
||||
// Merge all the skipped headers back
|
||||
for _, header := range skip {
|
||||
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
|
||||
}
|
||||
// Assemble and return the block download request
|
||||
if len(send) == 0 {
|
||||
return nil, process, nil
|
||||
}
|
||||
request := &fetchRequest{
|
||||
Peer: p,
|
||||
Headers: send,
|
||||
Time: time.Now(),
|
||||
}
|
||||
q.pendPool[p.id] = request
|
||||
|
||||
return request, process, nil
|
||||
}
|
||||
|
||||
// Cancel aborts a fetch request, returning all pending hashes to the queue.
|
||||
func (q *queue) Cancel(request *fetchRequest) {
|
||||
q.lock.Lock()
|
||||
@ -284,6 +381,9 @@ func (q *queue) Cancel(request *fetchRequest) {
|
||||
for hash, index := range request.Hashes {
|
||||
q.hashQueue.Push(hash, float32(index))
|
||||
}
|
||||
for _, header := range request.Headers {
|
||||
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
|
||||
}
|
||||
delete(q.pendPool, request.Peer.id)
|
||||
}
|
||||
|
||||
@ -310,8 +410,8 @@ func (q *queue) Expire(timeout time.Duration) []string {
|
||||
return peers
|
||||
}
|
||||
|
||||
// Deliver injects a block retrieval response into the download queue.
|
||||
func (q *queue) Deliver(id string, blocks []*types.Block) (err error) {
|
||||
// Deliver61 injects a block retrieval response into the download queue.
|
||||
func (q *queue) Deliver61(id string, blocks []*types.Block) (err error) {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
@ -337,19 +437,12 @@ func (q *queue) Deliver(id string, blocks []*types.Block) (err error) {
|
||||
errs = append(errs, fmt.Errorf("non-requested block %x", hash))
|
||||
continue
|
||||
}
|
||||
// If a requested block falls out of the range, the hash chain is invalid
|
||||
index := int(int64(block.NumberU64()) - int64(q.blockOffset))
|
||||
if index >= len(q.blockCache) || index < 0 {
|
||||
return errInvalidChain
|
||||
}
|
||||
// Otherwise merge the block and mark the hash block
|
||||
q.blockCache[index] = &Block{
|
||||
RawBlock: block,
|
||||
OriginPeer: id,
|
||||
// Queue the block up for processing
|
||||
if err := q.enqueue(id, block); err != nil {
|
||||
return err
|
||||
}
|
||||
delete(request.Hashes, hash)
|
||||
delete(q.hashPool, hash)
|
||||
q.blockPool[hash] = block.NumberU64()
|
||||
}
|
||||
// Return all failed or missing fetches to the queue
|
||||
for hash, index := range request.Hashes {
|
||||
@ -365,6 +458,88 @@ func (q *queue) Deliver(id string, blocks []*types.Block) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deliver injects a block body retrieval response into the download queue.
|
||||
func (q *queue) Deliver(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) error {
|
||||
q.lock.Lock()
|
||||
defer q.lock.Unlock()
|
||||
|
||||
// Short circuit if the block bodies were never requested
|
||||
request := q.pendPool[id]
|
||||
if request == nil {
|
||||
return errNoFetchesPending
|
||||
}
|
||||
delete(q.pendPool, id)
|
||||
|
||||
// If no block bodies were retrieved, mark them as unavailable for the origin peer
|
||||
if len(txLists) == 0 || len(uncleLists) == 0 {
|
||||
for hash, _ := range request.Headers {
|
||||
request.Peer.ignored.Add(hash)
|
||||
}
|
||||
}
|
||||
// Assemble each of the block bodies with their headers and queue for processing
|
||||
errs := make([]error, 0)
|
||||
for i, header := range request.Headers {
|
||||
// Short circuit block assembly if no more bodies are found
|
||||
if i >= len(txLists) || i >= len(uncleLists) {
|
||||
break
|
||||
}
|
||||
// Reconstruct the next block if contents match up
|
||||
if types.DeriveSha(types.Transactions(txLists[i])) != header.TxHash || types.CalcUncleHash(uncleLists[i]) != header.UncleHash {
|
||||
errs = []error{errInvalidBody}
|
||||
break
|
||||
}
|
||||
block := types.NewBlockWithHeader(header).WithBody(txLists[i], uncleLists[i])
|
||||
|
||||
// Queue the block up for processing
|
||||
if err := q.enqueue(id, block); err != nil {
|
||||
errs = []error{err}
|
||||
break
|
||||
}
|
||||
request.Headers[i] = nil
|
||||
delete(q.headerPool, header.Hash())
|
||||
}
|
||||
// Return all failed or missing fetches to the queue
|
||||
for _, header := range request.Headers {
|
||||
if header != nil {
|
||||
q.headerQueue.Push(header, -float32(header.Number.Uint64()))
|
||||
}
|
||||
}
|
||||
// If none of the blocks were good, it's a stale delivery
|
||||
switch {
|
||||
case len(errs) == 0:
|
||||
return nil
|
||||
|
||||
case len(errs) == 1 && errs[0] == errInvalidBody:
|
||||
return errInvalidBody
|
||||
|
||||
case len(errs) == 1 && errs[0] == errInvalidChain:
|
||||
return errInvalidChain
|
||||
|
||||
case len(errs) == len(request.Headers):
|
||||
return errStaleDelivery
|
||||
|
||||
default:
|
||||
return fmt.Errorf("multiple failures: %v", errs)
|
||||
}
|
||||
}
|
||||
|
||||
// enqueue inserts a new block into the final delivery queue, waiting for pickup
|
||||
// by the processor.
|
||||
func (q *queue) enqueue(origin string, block *types.Block) error {
|
||||
// If a requested block falls out of the range, the hash chain is invalid
|
||||
index := int(int64(block.NumberU64()) - int64(q.blockOffset))
|
||||
if index >= len(q.blockCache) || index < 0 {
|
||||
return errInvalidChain
|
||||
}
|
||||
// Otherwise merge the block and mark the hash done
|
||||
q.blockCache[index] = &Block{
|
||||
RawBlock: block,
|
||||
OriginPeer: origin,
|
||||
}
|
||||
q.blockPool[block.Header().Hash()] = block.NumberU64()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Prepare configures the block cache offset to allow accepting inbound blocks.
|
||||
func (q *queue) Prepare(offset uint64) {
|
||||
q.lock.Lock()
|
||||
|
@ -51,6 +51,12 @@ type blockRetrievalFn func(common.Hash) *types.Block
|
||||
// blockRequesterFn is a callback type for sending a block retrieval request.
|
||||
type blockRequesterFn func([]common.Hash) error
|
||||
|
||||
// headerRequesterFn is a callback type for sending a header retrieval request.
|
||||
type headerRequesterFn func(common.Hash) error
|
||||
|
||||
// bodyRequesterFn is a callback type for sending a body retrieval request.
|
||||
type bodyRequesterFn func([]common.Hash) error
|
||||
|
||||
// blockValidatorFn is a callback type to verify a block's header for fast propagation.
|
||||
type blockValidatorFn func(block *types.Block, parent *types.Block) error
|
||||
|
||||
@ -69,12 +75,30 @@ type peerDropFn func(id string)
|
||||
// announce is the hash notification of the availability of a new block in the
|
||||
// network.
|
||||
type announce struct {
|
||||
hash common.Hash // Hash of the block being announced
|
||||
number uint64 // Number of the block being announced (0 = unknown | old protocol)
|
||||
time time.Time // Timestamp of the announcement
|
||||
hash common.Hash // Hash of the block being announced
|
||||
number uint64 // Number of the block being announced (0 = unknown | old protocol)
|
||||
header *types.Header // Header of the block partially reassembled (new protocol)
|
||||
time time.Time // Timestamp of the announcement
|
||||
|
||||
origin string // Identifier of the peer originating the notification
|
||||
fetch blockRequesterFn // Fetcher function to retrieve
|
||||
origin string // Identifier of the peer originating the notification
|
||||
|
||||
fetch61 blockRequesterFn // [eth/61] Fetcher function to retrieve an announced block
|
||||
fetchHeader headerRequesterFn // [eth/62] Fetcher function to retrieve the header of an announced block
|
||||
fetchBodies bodyRequesterFn // [eth/62] Fetcher function to retrieve the body of an announced block
|
||||
}
|
||||
|
||||
// headerFilterTask represents a batch of headers needing fetcher filtering.
|
||||
type headerFilterTask struct {
|
||||
headers []*types.Header // Collection of headers to filter
|
||||
time time.Time // Arrival time of the headers
|
||||
}
|
||||
|
||||
// headerFilterTask represents a batch of block bodies (transactions and uncles)
|
||||
// needing fetcher filtering.
|
||||
type bodyFilterTask struct {
|
||||
transactions [][]*types.Transaction // Collection of transactions per block bodies
|
||||
uncles [][]*types.Header // Collection of uncles per block bodies
|
||||
time time.Time // Arrival time of the blocks' contents
|
||||
}
|
||||
|
||||
// inject represents a schedules import operation.
|
||||
@ -89,14 +113,20 @@ type Fetcher struct {
|
||||
// Various event channels
|
||||
notify chan *announce
|
||||
inject chan *inject
|
||||
filter chan chan []*types.Block
|
||||
done chan common.Hash
|
||||
quit chan struct{}
|
||||
|
||||
blockFilter chan chan []*types.Block
|
||||
headerFilter chan chan *headerFilterTask
|
||||
bodyFilter chan chan *bodyFilterTask
|
||||
|
||||
done chan common.Hash
|
||||
quit chan struct{}
|
||||
|
||||
// Announce states
|
||||
announces map[string]int // Per peer announce counts to prevent memory exhaustion
|
||||
announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching
|
||||
fetching map[common.Hash]*announce // Announced blocks, currently fetching
|
||||
announces map[string]int // Per peer announce counts to prevent memory exhaustion
|
||||
announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching
|
||||
fetching map[common.Hash]*announce // Announced blocks, currently fetching
|
||||
fetched map[common.Hash][]*announce // Blocks with headers fetched, scheduled for body retrieval
|
||||
completing map[common.Hash]*announce // Blocks with headers, currently body-completing
|
||||
|
||||
// Block cache
|
||||
queue *prque.Prque // Queue containing the import operations (block number sorted)
|
||||
@ -112,8 +142,9 @@ type Fetcher struct {
|
||||
dropPeer peerDropFn // Drops a peer for misbehaving
|
||||
|
||||
// Testing hooks
|
||||
fetchingHook func([]common.Hash) // Method to call upon starting a block fetch
|
||||
importedHook func(*types.Block) // Method to call upon successful block import
|
||||
fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch
|
||||
completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62)
|
||||
importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62)
|
||||
}
|
||||
|
||||
// New creates a block fetcher to retrieve blocks based on hash announcements.
|
||||
@ -121,12 +152,16 @@ func New(getBlock blockRetrievalFn, validateBlock blockValidatorFn, broadcastBlo
|
||||
return &Fetcher{
|
||||
notify: make(chan *announce),
|
||||
inject: make(chan *inject),
|
||||
filter: make(chan chan []*types.Block),
|
||||
blockFilter: make(chan chan []*types.Block),
|
||||
headerFilter: make(chan chan *headerFilterTask),
|
||||
bodyFilter: make(chan chan *bodyFilterTask),
|
||||
done: make(chan common.Hash),
|
||||
quit: make(chan struct{}),
|
||||
announces: make(map[string]int),
|
||||
announced: make(map[common.Hash][]*announce),
|
||||
fetching: make(map[common.Hash]*announce),
|
||||
fetched: make(map[common.Hash][]*announce),
|
||||
completing: make(map[common.Hash]*announce),
|
||||
queue: prque.New(),
|
||||
queues: make(map[string]int),
|
||||
queued: make(map[common.Hash]*inject),
|
||||
@ -153,13 +188,17 @@ func (f *Fetcher) Stop() {
|
||||
|
||||
// Notify announces the fetcher of the potential availability of a new block in
|
||||
// the network.
|
||||
func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, fetcher blockRequesterFn) error {
|
||||
func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time,
|
||||
blockFetcher blockRequesterFn, // eth/61 specific whole block fetcher
|
||||
headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error {
|
||||
block := &announce{
|
||||
hash: hash,
|
||||
number: number,
|
||||
time: time,
|
||||
origin: peer,
|
||||
fetch: fetcher,
|
||||
hash: hash,
|
||||
number: number,
|
||||
time: time,
|
||||
origin: peer,
|
||||
fetch61: blockFetcher,
|
||||
fetchHeader: headerFetcher,
|
||||
fetchBodies: bodyFetcher,
|
||||
}
|
||||
select {
|
||||
case f.notify <- block:
|
||||
@ -183,14 +222,16 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error {
|
||||
}
|
||||
}
|
||||
|
||||
// Filter extracts all the blocks that were explicitly requested by the fetcher,
|
||||
// FilterBlocks extracts all the blocks that were explicitly requested by the fetcher,
|
||||
// returning those that should be handled differently.
|
||||
func (f *Fetcher) Filter(blocks types.Blocks) types.Blocks {
|
||||
func (f *Fetcher) FilterBlocks(blocks types.Blocks) types.Blocks {
|
||||
glog.V(logger.Detail).Infof("[eth/61] filtering %d blocks", len(blocks))
|
||||
|
||||
// Send the filter channel to the fetcher
|
||||
filter := make(chan []*types.Block)
|
||||
|
||||
select {
|
||||
case f.filter <- filter:
|
||||
case f.blockFilter <- filter:
|
||||
case <-f.quit:
|
||||
return nil
|
||||
}
|
||||
@ -209,11 +250,69 @@ func (f *Fetcher) Filter(blocks types.Blocks) types.Blocks {
|
||||
}
|
||||
}
|
||||
|
||||
// FilterHeaders extracts all the headers that were explicitly requested by the fetcher,
|
||||
// returning those that should be handled differently.
|
||||
func (f *Fetcher) FilterHeaders(headers []*types.Header, time time.Time) []*types.Header {
|
||||
glog.V(logger.Detail).Infof("[eth/62] filtering %d headers", len(headers))
|
||||
|
||||
// Send the filter channel to the fetcher
|
||||
filter := make(chan *headerFilterTask)
|
||||
|
||||
select {
|
||||
case f.headerFilter <- filter:
|
||||
case <-f.quit:
|
||||
return nil
|
||||
}
|
||||
// Request the filtering of the header list
|
||||
select {
|
||||
case filter <- &headerFilterTask{headers: headers, time: time}:
|
||||
case <-f.quit:
|
||||
return nil
|
||||
}
|
||||
// Retrieve the headers remaining after filtering
|
||||
select {
|
||||
case task := <-filter:
|
||||
return task.headers
|
||||
case <-f.quit:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// FilterBodies extracts all the block bodies that were explicitly requested by
|
||||
// the fetcher, returning those that should be handled differently.
|
||||
func (f *Fetcher) FilterBodies(transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) {
|
||||
glog.V(logger.Detail).Infof("[eth/62] filtering %d:%d bodies", len(transactions), len(uncles))
|
||||
|
||||
// Send the filter channel to the fetcher
|
||||
filter := make(chan *bodyFilterTask)
|
||||
|
||||
select {
|
||||
case f.bodyFilter <- filter:
|
||||
case <-f.quit:
|
||||
return nil, nil
|
||||
}
|
||||
// Request the filtering of the body list
|
||||
select {
|
||||
case filter <- &bodyFilterTask{transactions: transactions, uncles: uncles, time: time}:
|
||||
case <-f.quit:
|
||||
return nil, nil
|
||||
}
|
||||
// Retrieve the bodies remaining after filtering
|
||||
select {
|
||||
case task := <-filter:
|
||||
return task.transactions, task.uncles
|
||||
case <-f.quit:
|
||||
return nil, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Loop is the main fetcher loop, checking and processing various notification
|
||||
// events.
|
||||
func (f *Fetcher) loop() {
|
||||
// Iterate the block fetching until a quit is requested
|
||||
fetch := time.NewTimer(0)
|
||||
fetchTimer := time.NewTimer(0)
|
||||
completeTimer := time.NewTimer(0)
|
||||
|
||||
for {
|
||||
// Clean up any expired block fetches
|
||||
for hash, announce := range f.fetching {
|
||||
@ -255,14 +354,25 @@ func (f *Fetcher) loop() {
|
||||
glog.V(logger.Debug).Infof("Peer %s: exceeded outstanding announces (%d)", notification.origin, hashLimit)
|
||||
break
|
||||
}
|
||||
// If we have a valid block number, check that it's potentially useful
|
||||
if notification.number > 0 {
|
||||
if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
|
||||
glog.V(logger.Debug).Infof("[eth/62] Peer %s: discarded announcement #%d [%x…], distance %d", notification.origin, notification.number, notification.hash[:4], dist)
|
||||
discardMeter.Mark(1)
|
||||
break
|
||||
}
|
||||
}
|
||||
// All is well, schedule the announce if block's not yet downloading
|
||||
if _, ok := f.fetching[notification.hash]; ok {
|
||||
break
|
||||
}
|
||||
if _, ok := f.completing[notification.hash]; ok {
|
||||
break
|
||||
}
|
||||
f.announces[notification.origin] = count
|
||||
f.announced[notification.hash] = append(f.announced[notification.hash], notification)
|
||||
if len(f.announced) == 1 {
|
||||
f.reschedule(fetch)
|
||||
f.rescheduleFetch(fetchTimer)
|
||||
}
|
||||
|
||||
case op := <-f.inject:
|
||||
@ -275,7 +385,7 @@ func (f *Fetcher) loop() {
|
||||
f.forgetHash(hash)
|
||||
f.forgetBlock(hash)
|
||||
|
||||
case <-fetch.C:
|
||||
case <-fetchTimer.C:
|
||||
// At least one block's timer ran out, check for needing retrieval
|
||||
request := make(map[string][]common.Hash)
|
||||
|
||||
@ -292,30 +402,77 @@ func (f *Fetcher) loop() {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Send out all block requests
|
||||
// Send out all block (eth/61) or header (eth/62) requests
|
||||
for peer, hashes := range request {
|
||||
if glog.V(logger.Detail) && len(hashes) > 0 {
|
||||
list := "["
|
||||
for _, hash := range hashes {
|
||||
list += fmt.Sprintf("%x, ", hash[:4])
|
||||
list += fmt.Sprintf("%x…, ", hash[:4])
|
||||
}
|
||||
list = list[:len(list)-2] + "]"
|
||||
|
||||
glog.V(logger.Detail).Infof("Peer %s: fetching %s", peer, list)
|
||||
if f.fetching[hashes[0]].fetch61 != nil {
|
||||
glog.V(logger.Detail).Infof("[eth/61] Peer %s: fetching blocks %s", peer, list)
|
||||
} else {
|
||||
glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching headers %s", peer, list)
|
||||
}
|
||||
}
|
||||
// Create a closure of the fetch and schedule in on a new thread
|
||||
fetcher, hashes := f.fetching[hashes[0]].fetch, hashes
|
||||
fetchBlocks, fetchHeader, hashes := f.fetching[hashes[0]].fetch61, f.fetching[hashes[0]].fetchHeader, hashes
|
||||
go func() {
|
||||
if f.fetchingHook != nil {
|
||||
f.fetchingHook(hashes)
|
||||
}
|
||||
fetcher(hashes)
|
||||
if fetchBlocks != nil {
|
||||
// Use old eth/61 protocol to retrieve whole blocks
|
||||
fetchBlocks(hashes)
|
||||
} else {
|
||||
// Use new eth/62 protocol to retrieve headers first
|
||||
for _, hash := range hashes {
|
||||
fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
// Schedule the next fetch if blocks are still pending
|
||||
f.reschedule(fetch)
|
||||
f.rescheduleFetch(fetchTimer)
|
||||
|
||||
case filter := <-f.filter:
|
||||
case <-completeTimer.C:
|
||||
// At least one header's timer ran out, retrieve everything
|
||||
request := make(map[string][]common.Hash)
|
||||
|
||||
for hash, announces := range f.fetched {
|
||||
// Pick a random peer to retrieve from, reset all others
|
||||
announce := announces[rand.Intn(len(announces))]
|
||||
f.forgetHash(hash)
|
||||
|
||||
// If the block still didn't arrive, queue for completion
|
||||
if f.getBlock(hash) == nil {
|
||||
request[announce.origin] = append(request[announce.origin], hash)
|
||||
f.completing[hash] = announce
|
||||
}
|
||||
}
|
||||
// Send out all block body requests
|
||||
for peer, hashes := range request {
|
||||
if glog.V(logger.Detail) && len(hashes) > 0 {
|
||||
list := "["
|
||||
for _, hash := range hashes {
|
||||
list += fmt.Sprintf("%x…, ", hash[:4])
|
||||
}
|
||||
list = list[:len(list)-2] + "]"
|
||||
|
||||
glog.V(logger.Detail).Infof("[eth/62] Peer %s: fetching bodies %s", peer, list)
|
||||
}
|
||||
// Create a closure of the fetch and schedule in on a new thread
|
||||
if f.completingHook != nil {
|
||||
f.completingHook(hashes)
|
||||
}
|
||||
go f.completing[hashes[0]].fetchBodies(hashes)
|
||||
}
|
||||
// Schedule the next fetch if blocks are still pending
|
||||
f.rescheduleComplete(completeTimer)
|
||||
|
||||
case filter := <-f.blockFilter:
|
||||
// Blocks arrived, extract any explicit fetches, return all else
|
||||
var blocks types.Blocks
|
||||
select {
|
||||
@ -352,12 +509,135 @@ func (f *Fetcher) loop() {
|
||||
f.enqueue(announce.origin, block)
|
||||
}
|
||||
}
|
||||
|
||||
case filter := <-f.headerFilter:
|
||||
// Headers arrived from a remote peer. Extract those that were explicitly
|
||||
// requested by the fetcher, and return everything else so it's delivered
|
||||
// to other parts of the system.
|
||||
var task *headerFilterTask
|
||||
select {
|
||||
case task = <-filter:
|
||||
case <-f.quit:
|
||||
return
|
||||
}
|
||||
// Split the batch of headers into unknown ones (to return to the caller),
|
||||
// known incomplete ones (requiring body retrievals) and completed blocks.
|
||||
unknown, incomplete, complete := []*types.Header{}, []*announce{}, []*types.Block{}
|
||||
for _, header := range task.headers {
|
||||
hash := header.Hash()
|
||||
|
||||
// Filter fetcher-requested headers from other synchronisation algorithms
|
||||
if announce := f.fetching[hash]; announce != nil && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil {
|
||||
// If the delivered header does not match the promised number, drop the announcer
|
||||
if header.Number.Uint64() != announce.number {
|
||||
glog.V(logger.Detail).Infof("[eth/62] Peer %s: invalid block number for [%x…]: announced %d, provided %d", announce.origin, header.Hash().Bytes()[:4], announce.number, header.Number.Uint64())
|
||||
f.dropPeer(announce.origin)
|
||||
f.forgetHash(hash)
|
||||
continue
|
||||
}
|
||||
// Only keep if not imported by other means
|
||||
if f.getBlock(hash) == nil {
|
||||
announce.header = header
|
||||
announce.time = task.time
|
||||
|
||||
// If the block is empty (header only), short circuit into the final import queue
|
||||
if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) {
|
||||
glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%x…] empty, skipping body retrieval", announce.origin, header.Number.Uint64(), header.Hash().Bytes()[:4])
|
||||
|
||||
complete = append(complete, types.NewBlockWithHeader(header))
|
||||
f.completing[hash] = announce
|
||||
continue
|
||||
}
|
||||
// Otherwise add to the list of blocks needing completion
|
||||
incomplete = append(incomplete, announce)
|
||||
} else {
|
||||
glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%x…] already imported, discarding header", announce.origin, header.Number.Uint64(), header.Hash().Bytes()[:4])
|
||||
f.forgetHash(hash)
|
||||
}
|
||||
} else {
|
||||
// Fetcher doesn't know about it, add to the return list
|
||||
unknown = append(unknown, header)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case filter <- &headerFilterTask{headers: unknown, time: task.time}:
|
||||
case <-f.quit:
|
||||
return
|
||||
}
|
||||
// Schedule the retrieved headers for body completion
|
||||
for _, announce := range incomplete {
|
||||
hash := announce.header.Hash()
|
||||
if _, ok := f.completing[hash]; ok {
|
||||
continue
|
||||
}
|
||||
f.fetched[hash] = append(f.fetched[hash], announce)
|
||||
if len(f.fetched) == 1 {
|
||||
f.rescheduleComplete(completeTimer)
|
||||
}
|
||||
}
|
||||
// Schedule the header-only blocks for import
|
||||
for _, block := range complete {
|
||||
if announce := f.completing[block.Hash()]; announce != nil {
|
||||
f.enqueue(announce.origin, block)
|
||||
}
|
||||
}
|
||||
|
||||
case filter := <-f.bodyFilter:
|
||||
// Block bodies arrived, extract any explicitly requested blocks, return the rest
|
||||
var task *bodyFilterTask
|
||||
select {
|
||||
case task = <-filter:
|
||||
case <-f.quit:
|
||||
return
|
||||
}
|
||||
|
||||
blocks := []*types.Block{}
|
||||
for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ {
|
||||
// Match up a body to any possible completion request
|
||||
matched := false
|
||||
|
||||
for hash, announce := range f.completing {
|
||||
if f.queued[hash] == nil {
|
||||
txnHash := types.DeriveSha(types.Transactions(task.transactions[i]))
|
||||
uncleHash := types.CalcUncleHash(task.uncles[i])
|
||||
|
||||
if txnHash == announce.header.TxHash && uncleHash == announce.header.UncleHash {
|
||||
// Mark the body matched, reassemble if still unknown
|
||||
matched = true
|
||||
|
||||
if f.getBlock(hash) == nil {
|
||||
blocks = append(blocks, types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]))
|
||||
} else {
|
||||
f.forgetHash(hash)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if matched {
|
||||
task.transactions = append(task.transactions[:i], task.transactions[i+1:]...)
|
||||
task.uncles = append(task.uncles[:i], task.uncles[i+1:]...)
|
||||
i--
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case filter <- task:
|
||||
case <-f.quit:
|
||||
return
|
||||
}
|
||||
// Schedule the retrieved blocks for ordered import
|
||||
for _, block := range blocks {
|
||||
if announce := f.completing[block.Hash()]; announce != nil {
|
||||
f.enqueue(announce.origin, block)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reschedule resets the specified fetch timer to the next announce timeout.
|
||||
func (f *Fetcher) reschedule(fetch *time.Timer) {
|
||||
// rescheduleFetch resets the specified fetch timer to the next announce timeout.
|
||||
func (f *Fetcher) rescheduleFetch(fetch *time.Timer) {
|
||||
// Short circuit if no blocks are announced
|
||||
if len(f.announced) == 0 {
|
||||
return
|
||||
@ -372,6 +652,22 @@ func (f *Fetcher) reschedule(fetch *time.Timer) {
|
||||
fetch.Reset(arriveTimeout - time.Since(earliest))
|
||||
}
|
||||
|
||||
// rescheduleComplete resets the specified completion timer to the next fetch timeout.
|
||||
func (f *Fetcher) rescheduleComplete(complete *time.Timer) {
|
||||
// Short circuit if no headers are fetched
|
||||
if len(f.fetched) == 0 {
|
||||
return
|
||||
}
|
||||
// Otherwise find the earliest expiring announcement
|
||||
earliest := time.Now()
|
||||
for _, announces := range f.fetched {
|
||||
if earliest.After(announces[0].time) {
|
||||
earliest = announces[0].time
|
||||
}
|
||||
}
|
||||
complete.Reset(gatherSlack - time.Since(earliest))
|
||||
}
|
||||
|
||||
// enqueue schedules a new future import operation, if the block to be imported
|
||||
// has not yet been seen.
|
||||
func (f *Fetcher) enqueue(peer string, block *types.Block) {
|
||||
@ -380,13 +676,15 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
|
||||
// Ensure the peer isn't DOSing us
|
||||
count := f.queues[peer] + 1
|
||||
if count > blockLimit {
|
||||
glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x], exceeded allowance (%d)", peer, block.NumberU64(), hash.Bytes()[:4], blockLimit)
|
||||
glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x…], exceeded allowance (%d)", peer, block.NumberU64(), hash.Bytes()[:4], blockLimit)
|
||||
f.forgetHash(hash)
|
||||
return
|
||||
}
|
||||
// Discard any past or too distant blocks
|
||||
if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist {
|
||||
glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x], distance %d", peer, block.NumberU64(), hash.Bytes()[:4], dist)
|
||||
glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x…], distance %d", peer, block.NumberU64(), hash.Bytes()[:4], dist)
|
||||
discardMeter.Mark(1)
|
||||
f.forgetHash(hash)
|
||||
return
|
||||
}
|
||||
// Schedule the block for future importing
|
||||
@ -400,7 +698,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) {
|
||||
f.queue.Push(op, -float32(block.NumberU64()))
|
||||
|
||||
if glog.V(logger.Debug) {
|
||||
glog.Infof("Peer %s: queued block #%d [%x], total %v", peer, block.NumberU64(), hash.Bytes()[:4], f.queue.Size())
|
||||
glog.Infof("Peer %s: queued block #%d [%x…], total %v", peer, block.NumberU64(), hash.Bytes()[:4], f.queue.Size())
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -412,13 +710,14 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
|
||||
hash := block.Hash()
|
||||
|
||||
// Run the import on a new thread
|
||||
glog.V(logger.Debug).Infof("Peer %s: importing block #%d [%x]", peer, block.NumberU64(), hash[:4])
|
||||
glog.V(logger.Debug).Infof("Peer %s: importing block #%d [%x…]", peer, block.NumberU64(), hash[:4])
|
||||
go func() {
|
||||
defer func() { f.done <- hash }()
|
||||
|
||||
// If the parent's unknown, abort insertion
|
||||
parent := f.getBlock(block.ParentHash())
|
||||
if parent == nil {
|
||||
glog.V(logger.Debug).Infof("Peer %s: parent []%x] of block #%d [%x…] unknown", block.ParentHash().Bytes()[:4], peer, block.NumberU64(), hash[:4])
|
||||
return
|
||||
}
|
||||
// Quickly validate the header and propagate the block if it passes
|
||||
@ -434,13 +733,13 @@ func (f *Fetcher) insert(peer string, block *types.Block) {
|
||||
|
||||
default:
|
||||
// Something went very wrong, drop the peer
|
||||
glog.V(logger.Debug).Infof("Peer %s: block #%d [%x] verification failed: %v", peer, block.NumberU64(), hash[:4], err)
|
||||
glog.V(logger.Debug).Infof("Peer %s: block #%d [%x…] verification failed: %v", peer, block.NumberU64(), hash[:4], err)
|
||||
f.dropPeer(peer)
|
||||
return
|
||||
}
|
||||
// Run the actual import and log any issues
|
||||
if _, err := f.insertChain(types.Blocks{block}); err != nil {
|
||||
glog.V(logger.Warn).Infof("Peer %s: block #%d [%x] import failed: %v", peer, block.NumberU64(), hash[:4], err)
|
||||
glog.V(logger.Warn).Infof("Peer %s: block #%d [%x…] import failed: %v", peer, block.NumberU64(), hash[:4], err)
|
||||
return
|
||||
}
|
||||
// If import succeeded, broadcast the block
|
||||
@ -474,9 +773,27 @@ func (f *Fetcher) forgetHash(hash common.Hash) {
|
||||
}
|
||||
delete(f.fetching, hash)
|
||||
}
|
||||
|
||||
// Remove any pending completion requests and decrement the DOS counters
|
||||
for _, announce := range f.fetched[hash] {
|
||||
f.announces[announce.origin]--
|
||||
if f.announces[announce.origin] == 0 {
|
||||
delete(f.announces, announce.origin)
|
||||
}
|
||||
}
|
||||
delete(f.fetched, hash)
|
||||
|
||||
// Remove any pending completions and decrement the DOS counters
|
||||
if announce := f.completing[hash]; announce != nil {
|
||||
f.announces[announce.origin]--
|
||||
if f.announces[announce.origin] == 0 {
|
||||
delete(f.announces, announce.origin)
|
||||
}
|
||||
delete(f.completing, hash)
|
||||
}
|
||||
}
|
||||
|
||||
// forgetBlock removes all traces of a queued block frmo the fetcher's internal
|
||||
// forgetBlock removes all traces of a queued block from the fetcher's internal
|
||||
// state.
|
||||
func (f *Fetcher) forgetBlock(hash common.Hash) {
|
||||
if insert := f.queued[hash]; insert != nil {
|
||||
|
@ -27,21 +27,39 @@ import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
)
|
||||
|
||||
var (
|
||||
testdb, _ = ethdb.NewMemDatabase()
|
||||
genesis = core.GenesisBlockForTesting(testdb, common.Address{}, big.NewInt(0))
|
||||
testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
|
||||
testAddress = crypto.PubkeyToAddress(testKey.PublicKey)
|
||||
genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000))
|
||||
unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit}, nil, nil, nil)
|
||||
)
|
||||
|
||||
// makeChain creates a chain of n blocks starting at and including parent.
|
||||
// the returned hash chain is ordered head->parent.
|
||||
// the returned hash chain is ordered head->parent. In addition, every 3rd block
|
||||
// contains a transaction and every 5th an uncle to allow testing correct block
|
||||
// reassembly.
|
||||
func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) {
|
||||
blocks := core.GenerateChain(parent, testdb, n, func(i int, gen *core.BlockGen) {
|
||||
gen.SetCoinbase(common.Address{seed})
|
||||
blocks := core.GenerateChain(parent, testdb, n, func(i int, block *core.BlockGen) {
|
||||
block.SetCoinbase(common.Address{seed})
|
||||
|
||||
// If the block number is multiple of 3, send a bonus transaction to the miner
|
||||
if parent == genesis && i%3 == 0 {
|
||||
tx, err := types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, nil, nil).SignECDSA(testKey)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
block.AddTx(tx)
|
||||
}
|
||||
// If the block number is a multiple of 5, add a bonus uncle to the block
|
||||
if i%5 == 0 {
|
||||
block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))})
|
||||
}
|
||||
})
|
||||
hashes := make([]common.Hash, n+1)
|
||||
hashes[len(hashes)-1] = parent.Hash()
|
||||
@ -60,6 +78,7 @@ type fetcherTester struct {
|
||||
|
||||
hashes []common.Hash // Hash chain belonging to the tester
|
||||
blocks map[common.Hash]*types.Block // Blocks belonging to the tester
|
||||
drops map[string]bool // Map of peers dropped by the fetcher
|
||||
|
||||
lock sync.RWMutex
|
||||
}
|
||||
@ -69,6 +88,7 @@ func newTester() *fetcherTester {
|
||||
tester := &fetcherTester{
|
||||
hashes: []common.Hash{genesis.Hash()},
|
||||
blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis},
|
||||
drops: make(map[string]bool),
|
||||
}
|
||||
tester.fetcher = New(tester.getBlock, tester.verifyBlock, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer)
|
||||
tester.fetcher.Start()
|
||||
@ -122,12 +142,14 @@ func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
// dropPeer is a nop placeholder for the peer removal.
|
||||
// dropPeer is an emulator for the peer removal, simply accumulating the various
|
||||
// peers dropped by the fetcher.
|
||||
func (f *fetcherTester) dropPeer(peer string) {
|
||||
f.drops[peer] = true
|
||||
}
|
||||
|
||||
// peerFetcher retrieves a fetcher associated with a simulated peer.
|
||||
func (f *fetcherTester) makeFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn {
|
||||
// makeBlockFetcher retrieves a block fetcher associated with a simulated peer.
|
||||
func (f *fetcherTester) makeBlockFetcher(blocks map[common.Hash]*types.Block) blockRequesterFn {
|
||||
closure := make(map[common.Hash]*types.Block)
|
||||
for hash, block := range blocks {
|
||||
closure[hash] = block
|
||||
@ -142,18 +164,105 @@ func (f *fetcherTester) makeFetcher(blocks map[common.Hash]*types.Block) blockRe
|
||||
}
|
||||
}
|
||||
// Return on a new thread
|
||||
go f.fetcher.Filter(blocks)
|
||||
go f.fetcher.FilterBlocks(blocks)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer.
|
||||
func (f *fetcherTester) makeHeaderFetcher(blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn {
|
||||
closure := make(map[common.Hash]*types.Block)
|
||||
for hash, block := range blocks {
|
||||
closure[hash] = block
|
||||
}
|
||||
// Create a function that return a header from the closure
|
||||
return func(hash common.Hash) error {
|
||||
// Gather the blocks to return
|
||||
headers := make([]*types.Header, 0, 1)
|
||||
if block, ok := closure[hash]; ok {
|
||||
headers = append(headers, block.Header())
|
||||
}
|
||||
// Return on a new thread
|
||||
go f.fetcher.FilterHeaders(headers, time.Now().Add(drift))
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer.
|
||||
func (f *fetcherTester) makeBodyFetcher(blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn {
|
||||
closure := make(map[common.Hash]*types.Block)
|
||||
for hash, block := range blocks {
|
||||
closure[hash] = block
|
||||
}
|
||||
// Create a function that returns blocks from the closure
|
||||
return func(hashes []common.Hash) error {
|
||||
// Gather the block bodies to return
|
||||
transactions := make([][]*types.Transaction, 0, len(hashes))
|
||||
uncles := make([][]*types.Header, 0, len(hashes))
|
||||
|
||||
for _, hash := range hashes {
|
||||
if block, ok := closure[hash]; ok {
|
||||
transactions = append(transactions, block.Transactions())
|
||||
uncles = append(uncles, block.Uncles())
|
||||
}
|
||||
}
|
||||
// Return on a new thread
|
||||
go f.fetcher.FilterBodies(transactions, uncles, time.Now().Add(drift))
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// verifyFetchingEvent verifies that one single event arrive on an fetching channel.
|
||||
func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) {
|
||||
if arrive {
|
||||
select {
|
||||
case <-fetching:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("fetching timeout")
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case <-fetching:
|
||||
t.Fatalf("fetching invoked")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verifyCompletingEvent verifies that one single event arrive on an completing channel.
|
||||
func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) {
|
||||
if arrive {
|
||||
select {
|
||||
case <-completing:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("completing timeout")
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case <-completing:
|
||||
t.Fatalf("completing invoked")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// verifyImportEvent verifies that one single event arrive on an import channel.
|
||||
func verifyImportEvent(t *testing.T, imported chan *types.Block) {
|
||||
select {
|
||||
case <-imported:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("import timeout")
|
||||
func verifyImportEvent(t *testing.T, imported chan *types.Block, arrive bool) {
|
||||
if arrive {
|
||||
select {
|
||||
case <-imported:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("import timeout")
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case <-imported:
|
||||
t.Fatalf("import invoked")
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -164,7 +273,7 @@ func verifyImportCount(t *testing.T, imported chan *types.Block, count int) {
|
||||
select {
|
||||
case <-imported:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("block %d: import timeout", i)
|
||||
t.Fatalf("block %d: import timeout", i+1)
|
||||
}
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
@ -181,51 +290,78 @@ func verifyImportDone(t *testing.T, imported chan *types.Block) {
|
||||
|
||||
// Tests that a fetcher accepts block announcements and initiates retrievals for
|
||||
// them, successfully importing into the local chain.
|
||||
func TestSequentialAnnouncements(t *testing.T) {
|
||||
func TestSequentialAnnouncements61(t *testing.T) { testSequentialAnnouncements(t, 61) }
|
||||
func TestSequentialAnnouncements62(t *testing.T) { testSequentialAnnouncements(t, 62) }
|
||||
func TestSequentialAnnouncements63(t *testing.T) { testSequentialAnnouncements(t, 63) }
|
||||
func TestSequentialAnnouncements64(t *testing.T) { testSequentialAnnouncements(t, 64) }
|
||||
|
||||
func testSequentialAnnouncements(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import
|
||||
targetBlocks := 4 * hashLimit
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
// Iteratively announce blocks until all are imported
|
||||
imported := make(chan *types.Block)
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
for i := len(hashes) - 2; i >= 0; i-- {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
verifyImportEvent(t, imported)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
verifyImportEvent(t, imported, true)
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
}
|
||||
|
||||
// Tests that if blocks are announced by multiple peers (or even the same buggy
|
||||
// peer), they will only get downloaded at most once.
|
||||
func TestConcurrentAnnouncements(t *testing.T) {
|
||||
func TestConcurrentAnnouncements61(t *testing.T) { testConcurrentAnnouncements(t, 61) }
|
||||
func TestConcurrentAnnouncements62(t *testing.T) { testConcurrentAnnouncements(t, 62) }
|
||||
func TestConcurrentAnnouncements63(t *testing.T) { testConcurrentAnnouncements(t, 63) }
|
||||
func TestConcurrentAnnouncements64(t *testing.T) { testConcurrentAnnouncements(t, 64) }
|
||||
|
||||
func testConcurrentAnnouncements(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import
|
||||
targetBlocks := 4 * hashLimit
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
// Assemble a tester with a built in counter for the requests
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
counter := uint32(0)
|
||||
wrapper := func(hashes []common.Hash) error {
|
||||
blockWrapper := func(hashes []common.Hash) error {
|
||||
atomic.AddUint32(&counter, uint32(len(hashes)))
|
||||
return fetcher(hashes)
|
||||
return blockFetcher(hashes)
|
||||
}
|
||||
headerWrapper := func(hash common.Hash) error {
|
||||
atomic.AddUint32(&counter, 1)
|
||||
return headerFetcher(hash)
|
||||
}
|
||||
// Iteratively announce blocks until all are imported
|
||||
imported := make(chan *types.Block)
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
for i := len(hashes) - 2; i >= 0; i-- {
|
||||
tester.fetcher.Notify("first", hashes[i], 0, time.Now().Add(-arriveTimeout), wrapper)
|
||||
tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout+time.Millisecond), wrapper)
|
||||
tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout-time.Millisecond), wrapper)
|
||||
|
||||
verifyImportEvent(t, imported)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("first", hashes[i], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil)
|
||||
tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout+time.Millisecond), blockWrapper, nil, nil)
|
||||
tester.fetcher.Notify("second", hashes[i], 0, time.Now().Add(-arriveTimeout-time.Millisecond), blockWrapper, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher)
|
||||
tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), nil, headerWrapper, bodyFetcher)
|
||||
tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), nil, headerWrapper, bodyFetcher)
|
||||
}
|
||||
verifyImportEvent(t, imported, true)
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
|
||||
@ -237,56 +373,90 @@ func TestConcurrentAnnouncements(t *testing.T) {
|
||||
|
||||
// Tests that announcements arriving while a previous is being fetched still
|
||||
// results in a valid import.
|
||||
func TestOverlappingAnnouncements(t *testing.T) {
|
||||
func TestOverlappingAnnouncements61(t *testing.T) { testOverlappingAnnouncements(t, 61) }
|
||||
func TestOverlappingAnnouncements62(t *testing.T) { testOverlappingAnnouncements(t, 62) }
|
||||
func TestOverlappingAnnouncements63(t *testing.T) { testOverlappingAnnouncements(t, 63) }
|
||||
func TestOverlappingAnnouncements64(t *testing.T) { testOverlappingAnnouncements(t, 64) }
|
||||
|
||||
func testOverlappingAnnouncements(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import
|
||||
targetBlocks := 4 * hashLimit
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
// Iteratively announce blocks, but overlap them continuously
|
||||
fetching := make(chan []common.Hash)
|
||||
overlap := 16
|
||||
imported := make(chan *types.Block, len(hashes)-1)
|
||||
tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes }
|
||||
for i := 0; i < overlap; i++ {
|
||||
imported <- nil
|
||||
}
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
for i := len(hashes) - 2; i >= 0; i-- {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
select {
|
||||
case <-fetching:
|
||||
case <-imported:
|
||||
case <-time.After(time.Second):
|
||||
t.Fatalf("hash %d: announce timeout", len(hashes)-i)
|
||||
t.Fatalf("block %d: import timeout", len(hashes)-i)
|
||||
}
|
||||
}
|
||||
// Wait for all the imports to complete and check count
|
||||
verifyImportCount(t, imported, len(hashes)-1)
|
||||
verifyImportCount(t, imported, overlap)
|
||||
}
|
||||
|
||||
// Tests that announces already being retrieved will not be duplicated.
|
||||
func TestPendingDeduplication(t *testing.T) {
|
||||
func TestPendingDeduplication61(t *testing.T) { testPendingDeduplication(t, 61) }
|
||||
func TestPendingDeduplication62(t *testing.T) { testPendingDeduplication(t, 62) }
|
||||
func TestPendingDeduplication63(t *testing.T) { testPendingDeduplication(t, 63) }
|
||||
func TestPendingDeduplication64(t *testing.T) { testPendingDeduplication(t, 64) }
|
||||
|
||||
func testPendingDeduplication(t *testing.T, protocol int) {
|
||||
// Create a hash and corresponding block
|
||||
hashes, blocks := makeChain(1, 0, genesis)
|
||||
|
||||
// Assemble a tester with a built in counter and delayed fetcher
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
delay := 50 * time.Millisecond
|
||||
counter := uint32(0)
|
||||
wrapper := func(hashes []common.Hash) error {
|
||||
blockWrapper := func(hashes []common.Hash) error {
|
||||
atomic.AddUint32(&counter, uint32(len(hashes)))
|
||||
|
||||
// Simulate a long running fetch
|
||||
go func() {
|
||||
time.Sleep(delay)
|
||||
fetcher(hashes)
|
||||
blockFetcher(hashes)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
headerWrapper := func(hash common.Hash) error {
|
||||
atomic.AddUint32(&counter, 1)
|
||||
|
||||
// Simulate a long running fetch
|
||||
go func() {
|
||||
time.Sleep(delay)
|
||||
headerFetcher(hash)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
// Announce the same block many times until it's fetched (wait for any pending ops)
|
||||
for tester.getBlock(hashes[0]) == nil {
|
||||
tester.fetcher.Notify("repeater", hashes[0], 0, time.Now().Add(-arriveTimeout), wrapper)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("repeater", hashes[0], 0, time.Now().Add(-arriveTimeout), blockWrapper, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerWrapper, bodyFetcher)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
time.Sleep(delay)
|
||||
@ -302,14 +472,21 @@ func TestPendingDeduplication(t *testing.T) {
|
||||
|
||||
// Tests that announcements retrieved in a random order are cached and eventually
|
||||
// imported when all the gaps are filled in.
|
||||
func TestRandomArrivalImport(t *testing.T) {
|
||||
func TestRandomArrivalImport61(t *testing.T) { testRandomArrivalImport(t, 61) }
|
||||
func TestRandomArrivalImport62(t *testing.T) { testRandomArrivalImport(t, 62) }
|
||||
func TestRandomArrivalImport63(t *testing.T) { testRandomArrivalImport(t, 63) }
|
||||
func TestRandomArrivalImport64(t *testing.T) { testRandomArrivalImport(t, 64) }
|
||||
|
||||
func testRandomArrivalImport(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import, and choose one to delay
|
||||
targetBlocks := maxQueueDist
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
skip := targetBlocks / 2
|
||||
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
// Iteratively announce blocks, skipping one entry
|
||||
imported := make(chan *types.Block, len(hashes)-1)
|
||||
@ -317,25 +494,40 @@ func TestRandomArrivalImport(t *testing.T) {
|
||||
|
||||
for i := len(hashes) - 1; i >= 0; i-- {
|
||||
if i != skip {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
// Finally announce the skipped entry and check full import
|
||||
tester.fetcher.Notify("valid", hashes[skip], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[skip], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
verifyImportCount(t, imported, len(hashes)-1)
|
||||
}
|
||||
|
||||
// Tests that direct block enqueues (due to block propagation vs. hash announce)
|
||||
// are correctly schedule, filling and import queue gaps.
|
||||
func TestQueueGapFill(t *testing.T) {
|
||||
func TestQueueGapFill61(t *testing.T) { testQueueGapFill(t, 61) }
|
||||
func TestQueueGapFill62(t *testing.T) { testQueueGapFill(t, 62) }
|
||||
func TestQueueGapFill63(t *testing.T) { testQueueGapFill(t, 63) }
|
||||
func TestQueueGapFill64(t *testing.T) { testQueueGapFill(t, 64) }
|
||||
|
||||
func testQueueGapFill(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import, and choose one to not announce at all
|
||||
targetBlocks := maxQueueDist
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
skip := targetBlocks / 2
|
||||
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
// Iteratively announce blocks, skipping one entry
|
||||
imported := make(chan *types.Block, len(hashes)-1)
|
||||
@ -343,7 +535,11 @@ func TestQueueGapFill(t *testing.T) {
|
||||
|
||||
for i := len(hashes) - 1; i >= 0; i-- {
|
||||
if i != skip {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
@ -354,13 +550,20 @@ func TestQueueGapFill(t *testing.T) {
|
||||
|
||||
// Tests that blocks arriving from various sources (multiple propagations, hash
|
||||
// announces, etc) do not get scheduled for import multiple times.
|
||||
func TestImportDeduplication(t *testing.T) {
|
||||
func TestImportDeduplication61(t *testing.T) { testImportDeduplication(t, 61) }
|
||||
func TestImportDeduplication62(t *testing.T) { testImportDeduplication(t, 62) }
|
||||
func TestImportDeduplication63(t *testing.T) { testImportDeduplication(t, 63) }
|
||||
func TestImportDeduplication64(t *testing.T) { testImportDeduplication(t, 64) }
|
||||
|
||||
func testImportDeduplication(t *testing.T, protocol int) {
|
||||
// Create two blocks to import (one for duplication, the other for stalling)
|
||||
hashes, blocks := makeChain(2, 0, genesis)
|
||||
|
||||
// Create the tester and wrap the importer with a counter
|
||||
tester := newTester()
|
||||
fetcher := tester.makeFetcher(blocks)
|
||||
blockFetcher := tester.makeBlockFetcher(blocks)
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
counter := uint32(0)
|
||||
tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) {
|
||||
@ -374,7 +577,11 @@ func TestImportDeduplication(t *testing.T) {
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
// Announce the duplicating block, wait for retrieval, and also propagate directly
|
||||
tester.fetcher.Notify("valid", hashes[0], 0, time.Now().Add(-arriveTimeout), fetcher)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[0], 0, time.Now().Add(-arriveTimeout), blockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
}
|
||||
<-fetching
|
||||
|
||||
tester.fetcher.Enqueue("valid", blocks[hashes[0]])
|
||||
@ -391,35 +598,157 @@ func TestImportDeduplication(t *testing.T) {
|
||||
}
|
||||
|
||||
// Tests that blocks with numbers much lower or higher than out current head get
|
||||
// discarded no prevent wasting resources on useless blocks from faulty peers.
|
||||
func TestDistantDiscarding(t *testing.T) {
|
||||
// Create a long chain to import
|
||||
// discarded to prevent wasting resources on useless blocks from faulty peers.
|
||||
func TestDistantPropagationDiscarding(t *testing.T) {
|
||||
// Create a long chain to import and define the discard boundaries
|
||||
hashes, blocks := makeChain(3*maxQueueDist, 0, genesis)
|
||||
head := hashes[len(hashes)/2]
|
||||
|
||||
low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1
|
||||
|
||||
// Create a tester and simulate a head block being the middle of the above chain
|
||||
tester := newTester()
|
||||
tester.hashes = []common.Hash{head}
|
||||
tester.blocks = map[common.Hash]*types.Block{head: blocks[head]}
|
||||
|
||||
// Ensure that a block with a lower number than the threshold is discarded
|
||||
tester.fetcher.Enqueue("lower", blocks[hashes[0]])
|
||||
tester.fetcher.Enqueue("lower", blocks[hashes[low]])
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if !tester.fetcher.queue.Empty() {
|
||||
t.Fatalf("fetcher queued stale block")
|
||||
}
|
||||
// Ensure that a block with a higher number than the threshold is discarded
|
||||
tester.fetcher.Enqueue("higher", blocks[hashes[len(hashes)-1]])
|
||||
tester.fetcher.Enqueue("higher", blocks[hashes[high]])
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if !tester.fetcher.queue.Empty() {
|
||||
t.Fatalf("fetcher queued future block")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that announcements with numbers much lower or higher than out current
|
||||
// head get discarded to prevent wasting resources on useless blocks from faulty
|
||||
// peers.
|
||||
func TestDistantAnnouncementDiscarding62(t *testing.T) { testDistantAnnouncementDiscarding(t, 62) }
|
||||
func TestDistantAnnouncementDiscarding63(t *testing.T) { testDistantAnnouncementDiscarding(t, 63) }
|
||||
func TestDistantAnnouncementDiscarding64(t *testing.T) { testDistantAnnouncementDiscarding(t, 64) }
|
||||
|
||||
func testDistantAnnouncementDiscarding(t *testing.T, protocol int) {
|
||||
// Create a long chain to import and define the discard boundaries
|
||||
hashes, blocks := makeChain(3*maxQueueDist, 0, genesis)
|
||||
head := hashes[len(hashes)/2]
|
||||
|
||||
low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1
|
||||
|
||||
// Create a tester and simulate a head block being the middle of the above chain
|
||||
tester := newTester()
|
||||
tester.hashes = []common.Hash{head}
|
||||
tester.blocks = map[common.Hash]*types.Block{head: blocks[head]}
|
||||
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
fetching := make(chan struct{}, 2)
|
||||
tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} }
|
||||
|
||||
// Ensure that a block with a lower number than the threshold is discarded
|
||||
tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
select {
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
case <-fetching:
|
||||
t.Fatalf("fetcher requested stale header")
|
||||
}
|
||||
// Ensure that a block with a higher number than the threshold is discarded
|
||||
tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
select {
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
case <-fetching:
|
||||
t.Fatalf("fetcher requested future header")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests that peers announcing blocks with invalid numbers (i.e. not matching
|
||||
// the headers provided afterwards) get dropped as malicious.
|
||||
func TestInvalidNumberAnnouncement62(t *testing.T) { testInvalidNumberAnnouncement(t, 62) }
|
||||
func TestInvalidNumberAnnouncement63(t *testing.T) { testInvalidNumberAnnouncement(t, 63) }
|
||||
func TestInvalidNumberAnnouncement64(t *testing.T) { testInvalidNumberAnnouncement(t, 64) }
|
||||
|
||||
func testInvalidNumberAnnouncement(t *testing.T, protocol int) {
|
||||
// Create a single block to import and check numbers against
|
||||
hashes, blocks := makeChain(1, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
imported := make(chan *types.Block)
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
// Announce a block with a bad number, check for immediate drop
|
||||
tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
verifyImportEvent(t, imported, false)
|
||||
|
||||
if !tester.drops["bad"] {
|
||||
t.Fatalf("peer with invalid numbered announcement not dropped")
|
||||
}
|
||||
// Make sure a good announcement passes without a drop
|
||||
tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
verifyImportEvent(t, imported, true)
|
||||
|
||||
if tester.drops["good"] {
|
||||
t.Fatalf("peer with valid numbered announcement dropped")
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
}
|
||||
|
||||
// Tests that if a block is empty (i.e. header only), no body request should be
|
||||
// made, and instead the header should be assembled into a whole block in itself.
|
||||
func TestEmptyBlockShortCircuit62(t *testing.T) { testEmptyBlockShortCircuit(t, 62) }
|
||||
func TestEmptyBlockShortCircuit63(t *testing.T) { testEmptyBlockShortCircuit(t, 63) }
|
||||
func TestEmptyBlockShortCircuit64(t *testing.T) { testEmptyBlockShortCircuit(t, 64) }
|
||||
|
||||
func testEmptyBlockShortCircuit(t *testing.T, protocol int) {
|
||||
// Create a chain of blocks to import
|
||||
hashes, blocks := makeChain(32, 0, genesis)
|
||||
|
||||
tester := newTester()
|
||||
headerFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
bodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
// Add a monitoring hook for all internal events
|
||||
fetching := make(chan []common.Hash)
|
||||
tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes }
|
||||
|
||||
completing := make(chan []common.Hash)
|
||||
tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes }
|
||||
|
||||
imported := make(chan *types.Block)
|
||||
tester.fetcher.importedHook = func(block *types.Block) { imported <- block }
|
||||
|
||||
// Iteratively announce blocks until all are imported
|
||||
for i := len(hashes) - 2; i >= 0; i-- {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, headerFetcher, bodyFetcher)
|
||||
|
||||
// All announces should fetch the header
|
||||
verifyFetchingEvent(t, fetching, true)
|
||||
|
||||
// Only blocks with data contents should request bodies
|
||||
verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0)
|
||||
|
||||
// Irrelevant of the construct, import should succeed
|
||||
verifyImportEvent(t, imported, true)
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
}
|
||||
|
||||
// Tests that a peer is unable to use unbounded memory with sending infinite
|
||||
// block announcements to a node, but that even in the face of such an attack,
|
||||
// the fetcher remains operational.
|
||||
func TestHashMemoryExhaustionAttack(t *testing.T) {
|
||||
func TestHashMemoryExhaustionAttack61(t *testing.T) { testHashMemoryExhaustionAttack(t, 61) }
|
||||
func TestHashMemoryExhaustionAttack62(t *testing.T) { testHashMemoryExhaustionAttack(t, 62) }
|
||||
func TestHashMemoryExhaustionAttack63(t *testing.T) { testHashMemoryExhaustionAttack(t, 63) }
|
||||
func TestHashMemoryExhaustionAttack64(t *testing.T) { testHashMemoryExhaustionAttack(t, 64) }
|
||||
|
||||
func testHashMemoryExhaustionAttack(t *testing.T, protocol int) {
|
||||
// Create a tester with instrumented import hooks
|
||||
tester := newTester()
|
||||
|
||||
@ -429,17 +758,29 @@ func TestHashMemoryExhaustionAttack(t *testing.T) {
|
||||
// Create a valid chain and an infinite junk chain
|
||||
targetBlocks := hashLimit + 2*maxQueueDist
|
||||
hashes, blocks := makeChain(targetBlocks, 0, genesis)
|
||||
valid := tester.makeFetcher(blocks)
|
||||
validBlockFetcher := tester.makeBlockFetcher(blocks)
|
||||
validHeaderFetcher := tester.makeHeaderFetcher(blocks, -gatherSlack)
|
||||
validBodyFetcher := tester.makeBodyFetcher(blocks, 0)
|
||||
|
||||
attack, _ := makeChain(targetBlocks, 0, unknownBlock)
|
||||
attacker := tester.makeFetcher(nil)
|
||||
attackerBlockFetcher := tester.makeBlockFetcher(nil)
|
||||
attackerHeaderFetcher := tester.makeHeaderFetcher(nil, -gatherSlack)
|
||||
attackerBodyFetcher := tester.makeBodyFetcher(nil, 0)
|
||||
|
||||
// Feed the tester a huge hashset from the attacker, and a limited from the valid peer
|
||||
for i := 0; i < len(attack); i++ {
|
||||
if i < maxQueueDist {
|
||||
tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], 0, time.Now(), valid)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], 0, time.Now(), validBlockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), nil, validHeaderFetcher, validBodyFetcher)
|
||||
}
|
||||
}
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("attacker", attack[i], 0, time.Now(), attackerBlockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), nil, attackerHeaderFetcher, attackerBodyFetcher)
|
||||
}
|
||||
tester.fetcher.Notify("attacker", attack[i], 0, time.Now(), attacker)
|
||||
}
|
||||
if len(tester.fetcher.announced) != hashLimit+maxQueueDist {
|
||||
t.Fatalf("queued announce count mismatch: have %d, want %d", len(tester.fetcher.announced), hashLimit+maxQueueDist)
|
||||
@ -449,8 +790,12 @@ func TestHashMemoryExhaustionAttack(t *testing.T) {
|
||||
|
||||
// Feed the remaining valid hashes to ensure DOS protection state remains clean
|
||||
for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), valid)
|
||||
verifyImportEvent(t, imported)
|
||||
if protocol < 62 {
|
||||
tester.fetcher.Notify("valid", hashes[i], 0, time.Now().Add(-arriveTimeout), validBlockFetcher, nil, nil)
|
||||
} else {
|
||||
tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), nil, validHeaderFetcher, validBodyFetcher)
|
||||
}
|
||||
verifyImportEvent(t, imported, true)
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
}
|
||||
@ -498,7 +843,7 @@ func TestBlockMemoryExhaustionAttack(t *testing.T) {
|
||||
// Insert the remaining blocks in chunks to ensure clean DOS protection
|
||||
for i := maxQueueDist; i < len(hashes)-1; i++ {
|
||||
tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]])
|
||||
verifyImportEvent(t, imported)
|
||||
verifyImportEvent(t, imported, true)
|
||||
}
|
||||
verifyImportDone(t, imported)
|
||||
}
|
||||
|
@ -201,7 +201,9 @@ func (pm *ProtocolManager) handle(p *peer) error {
|
||||
defer pm.removePeer(p.id)
|
||||
|
||||
// Register the peer in the downloader. If the downloader considers it banned, we disconnect
|
||||
if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(), p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks); err != nil {
|
||||
if err := pm.downloader.RegisterPeer(p.id, p.version, p.Head(),
|
||||
p.RequestHashes, p.RequestHashesFromNumber, p.RequestBlocks,
|
||||
p.RequestHeadersByHash, p.RequestHeadersByNumber, p.RequestBodies); err != nil {
|
||||
return err
|
||||
}
|
||||
// Propagate existing transactions. new transactions appearing
|
||||
@ -287,7 +289,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
|
||||
break
|
||||
}
|
||||
// Deliver them all to the downloader for queuing
|
||||
err := pm.downloader.DeliverHashes(p.id, hashes)
|
||||
err := pm.downloader.DeliverHashes61(p.id, hashes)
|
||||
if err != nil {
|
||||
glog.V(logger.Debug).Infoln(err)
|
||||
}
|
||||
@ -332,8 +334,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
|
||||
block.ReceivedAt = msg.ReceivedAt
|
||||
}
|
||||
// Filter out any explicitly requested blocks, deliver the rest to the downloader
|
||||
if blocks := pm.fetcher.Filter(blocks); len(blocks) > 0 {
|
||||
pm.downloader.DeliverBlocks(p.id, blocks)
|
||||
if blocks := pm.fetcher.FilterBlocks(blocks); len(blocks) > 0 {
|
||||
pm.downloader.DeliverBlocks61(p.id, blocks)
|
||||
}
|
||||
|
||||
// Block header query, collect the requested headers and reply
|
||||
@ -401,6 +403,46 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
|
||||
}
|
||||
return p.SendBlockHeaders(headers)
|
||||
|
||||
case p.version >= eth62 && msg.Code == BlockHeadersMsg:
|
||||
// A batch of headers arrived to one of our previous requests
|
||||
var headers []*types.Header
|
||||
if err := msg.Decode(&headers); err != nil {
|
||||
return errResp(ErrDecode, "msg %v: %v", msg, err)
|
||||
}
|
||||
// Filter out any explicitly requested headers, deliver the rest to the downloader
|
||||
filter := len(headers) == 1
|
||||
if filter {
|
||||
headers = pm.fetcher.FilterHeaders(headers, time.Now())
|
||||
}
|
||||
if len(headers) > 0 || !filter {
|
||||
err := pm.downloader.DeliverHeaders(p.id, headers)
|
||||
if err != nil {
|
||||
glog.V(logger.Debug).Infoln(err)
|
||||
}
|
||||
}
|
||||
|
||||
case p.version >= eth62 && msg.Code == BlockBodiesMsg:
|
||||
// A batch of block bodies arrived to one of our previous requests
|
||||
var request blockBodiesData
|
||||
if err := msg.Decode(&request); err != nil {
|
||||
return errResp(ErrDecode, "msg %v: %v", msg, err)
|
||||
}
|
||||
// Deliver them all to the downloader for queuing
|
||||
trasactions := make([][]*types.Transaction, len(request))
|
||||
uncles := make([][]*types.Header, len(request))
|
||||
|
||||
for i, body := range request {
|
||||
trasactions[i] = body.Transactions
|
||||
uncles[i] = body.Uncles
|
||||
}
|
||||
// Filter out any explicitly requested bodies, deliver the rest to the downloader
|
||||
if trasactions, uncles := pm.fetcher.FilterBodies(trasactions, uncles, time.Now()); len(trasactions) > 0 || len(uncles) > 0 {
|
||||
err := pm.downloader.DeliverBodies(p.id, trasactions, uncles)
|
||||
if err != nil {
|
||||
glog.V(logger.Debug).Infoln(err)
|
||||
}
|
||||
}
|
||||
|
||||
case p.version >= eth62 && msg.Code == GetBlockBodiesMsg:
|
||||
// Decode the retrieval message
|
||||
msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size))
|
||||
@ -522,7 +564,11 @@ func (pm *ProtocolManager) handleMsg(p *peer) error {
|
||||
}
|
||||
}
|
||||
for _, block := range unknown {
|
||||
pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestBlocks)
|
||||
if p.version < eth62 {
|
||||
pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestBlocks, nil, nil)
|
||||
} else {
|
||||
pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), nil, p.RequestOneHeader, p.RequestBodies)
|
||||
}
|
||||
}
|
||||
|
||||
case msg.Code == NewBlockMsg:
|
||||
@ -612,7 +658,11 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) {
|
||||
// Otherwise if the block is indeed in out own chain, announce it
|
||||
if pm.chainman.HasBlock(hash) {
|
||||
for _, peer := range peers {
|
||||
peer.SendNewBlockHashes([]common.Hash{hash})
|
||||
if peer.version < eth62 {
|
||||
peer.SendNewBlockHashes61([]common.Hash{hash})
|
||||
} else {
|
||||
peer.SendNewBlockHashes([]common.Hash{hash}, []uint64{block.NumberU64()})
|
||||
}
|
||||
}
|
||||
glog.V(logger.Detail).Infof("announced block %x to %d peers in %v", hash[:4], len(peers), time.Since(block.ReceivedAt))
|
||||
}
|
||||
|
59
eth/peer.go
59
eth/peer.go
@ -145,15 +145,29 @@ func (p *peer) SendBlocks(blocks []*types.Block) error {
|
||||
return p2p.Send(p.rw, BlocksMsg, blocks)
|
||||
}
|
||||
|
||||
// SendNewBlockHashes announces the availability of a number of blocks through
|
||||
// SendNewBlockHashes61 announces the availability of a number of blocks through
|
||||
// a hash notification.
|
||||
func (p *peer) SendNewBlockHashes(hashes []common.Hash) error {
|
||||
func (p *peer) SendNewBlockHashes61(hashes []common.Hash) error {
|
||||
for _, hash := range hashes {
|
||||
p.knownBlocks.Add(hash)
|
||||
}
|
||||
return p2p.Send(p.rw, NewBlockHashesMsg, hashes)
|
||||
}
|
||||
|
||||
// SendNewBlockHashes announces the availability of a number of blocks through
|
||||
// a hash notification.
|
||||
func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error {
|
||||
for _, hash := range hashes {
|
||||
p.knownBlocks.Add(hash)
|
||||
}
|
||||
request := make(newBlockHashesData, len(hashes))
|
||||
for i := 0; i < len(hashes); i++ {
|
||||
request[i].Hash = hashes[i]
|
||||
request[i].Number = numbers[i]
|
||||
}
|
||||
return p2p.Send(p.rw, NewBlockHashesMsg, request)
|
||||
}
|
||||
|
||||
// SendNewBlock propagates an entire block to a remote peer.
|
||||
func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error {
|
||||
p.knownBlocks.Add(block.Hash())
|
||||
@ -185,40 +199,61 @@ func (p *peer) SendReceipts(receipts []*types.Receipt) error {
|
||||
// RequestHashes fetches a batch of hashes from a peer, starting at from, going
|
||||
// towards the genesis block.
|
||||
func (p *peer) RequestHashes(from common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching hashes (%d) from %x...\n", p, downloader.MaxHashFetch, from[:4])
|
||||
glog.V(logger.Debug).Infof("%v fetching hashes (%d) from %x...", p, downloader.MaxHashFetch, from[:4])
|
||||
return p2p.Send(p.rw, GetBlockHashesMsg, getBlockHashesData{from, uint64(downloader.MaxHashFetch)})
|
||||
}
|
||||
|
||||
// RequestHashesFromNumber fetches a batch of hashes from a peer, starting at
|
||||
// the requested block number, going upwards towards the genesis block.
|
||||
func (p *peer) RequestHashesFromNumber(from uint64, count int) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching hashes (%d) from #%d...\n", p, count, from)
|
||||
glog.V(logger.Debug).Infof("%v fetching hashes (%d) from #%d...", p, count, from)
|
||||
return p2p.Send(p.rw, GetBlockHashesFromNumberMsg, getBlockHashesFromNumberData{from, uint64(count)})
|
||||
}
|
||||
|
||||
// RequestBlocks fetches a batch of blocks corresponding to the specified hashes.
|
||||
func (p *peer) RequestBlocks(hashes []common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %v blocks\n", p, len(hashes))
|
||||
glog.V(logger.Debug).Infof("%v fetching %v blocks", p, len(hashes))
|
||||
return p2p.Send(p.rw, GetBlocksMsg, hashes)
|
||||
}
|
||||
|
||||
// RequestHeaders fetches a batch of blocks' headers corresponding to the
|
||||
// specified hashes.
|
||||
func (p *peer) RequestHeaders(hashes []common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %v headers\n", p, len(hashes))
|
||||
return p2p.Send(p.rw, GetBlockHeadersMsg, hashes)
|
||||
// RequestHeaders is a wrapper around the header query functions to fetch a
|
||||
// single header. It is used solely by the fetcher.
|
||||
func (p *peer) RequestOneHeader(hash common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching a single header: %x", p, hash)
|
||||
return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Hash: hash}, Amount: uint64(1), Skip: uint64(0), Reverse: false})
|
||||
}
|
||||
|
||||
// RequestHeadersByHash fetches a batch of blocks' headers corresponding to the
|
||||
// specified header query, based on the hash of an origin block.
|
||||
func (p *peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %d headers from %x, skipping %d (reverse = %v)", p, amount, origin[:4], skip, reverse)
|
||||
return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Hash: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse})
|
||||
}
|
||||
|
||||
// RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the
|
||||
// specified header query, based on the number of an origin block.
|
||||
func (p *peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %d headers from #%d, skipping %d (reverse = %v)", p, amount, origin, skip, reverse)
|
||||
return p2p.Send(p.rw, GetBlockHeadersMsg, &getBlockHeadersData{Origin: hashOrNumber{Number: origin}, Amount: uint64(amount), Skip: uint64(skip), Reverse: reverse})
|
||||
}
|
||||
|
||||
// RequestBodies fetches a batch of blocks' bodies corresponding to the hashes
|
||||
// specified.
|
||||
func (p *peer) RequestBodies(hashes []common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %d block bodies", p, len(hashes))
|
||||
return p2p.Send(p.rw, GetBlockBodiesMsg, hashes)
|
||||
}
|
||||
|
||||
// RequestNodeData fetches a batch of arbitrary data from a node's known state
|
||||
// data, corresponding to the specified hashes.
|
||||
func (p *peer) RequestNodeData(hashes []common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %v state data\n", p, len(hashes))
|
||||
glog.V(logger.Debug).Infof("%v fetching %v state data", p, len(hashes))
|
||||
return p2p.Send(p.rw, GetNodeDataMsg, hashes)
|
||||
}
|
||||
|
||||
// RequestReceipts fetches a batch of transaction receipts from a remote node.
|
||||
func (p *peer) RequestReceipts(hashes []common.Hash) error {
|
||||
glog.V(logger.Debug).Infof("%v fetching %v receipts\n", p, len(hashes))
|
||||
glog.V(logger.Debug).Infof("%v fetching %v receipts", p, len(hashes))
|
||||
return p2p.Send(p.rw, GetReceiptsMsg, hashes)
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user