roysc/plugeth
1
0
Fork 0
forked from cerc-io/plugeth
Code Pull Requests Activity

core, les, eth: port snap sync changes (#24898)

Browse Source 
core, eth, les, trie: rework snap sync
This commit is contained in:
rjl493456442
2022-07-15 14:55:51 +03:00
committed by GitHub
parent 1c9afc56ae
commit 1657e43931
12 changed files with 871 additions and 516 deletions
Download Patch File Download Diff File Expand all files Collapse all files
eth/downloader/skeleton_test.go
+1 -2
View File
@@ -21,7 +21,6 @@ import (
"errors"
"fmt"
"math/big"
"os"
"sync/atomic"
"testing"
"time"
@@ -515,7 +514,7 @@ func TestSkeletonSyncExtend(t *testing.T) {
// Tests that the skeleton sync correctly retrieves headers from one or more
// peers without duplicates or other strange side effects.
func TestSkeletonSyncRetrievals(t *testing.T) {
log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
//log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(true))))
// Since skeleton headers don't need to be meaningful, beyond a parent hash
// progression, create a long fake chain to test with.
eth/protocols/snap/sort_test.go
+9 -16
View File
@@ -22,7 +22,6 @@ import (
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/trie"
)
func hexToNibbles(s string) []byte {
@@ -38,22 +37,17 @@ func hexToNibbles(s string) []byte {
}
func TestRequestSorting(t *testing.T) {
// - Path 0x9 -> {0x19}
// - Path 0x99 -> {0x0099}
// - Path 0x01234567890123456789012345678901012345678901234567890123456789019 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x19}
// - Path 0x012345678901234567890123456789010123456789012345678901234567890199 -> {0x0123456789012345678901234567890101234567890123456789012345678901, 0x0099}
var f = func(path string) (trie.SyncPath, TrieNodePathSet, common.Hash) {
var f = func(path string) string {
data := hexToNibbles(path)
sp := trie.NewSyncPath(data)
tnps := TrieNodePathSet([][]byte(sp))
hash := common.Hash{}
return sp, tnps, hash
return string(data)
}
var (
hashes []common.Hash
paths []trie.SyncPath
pathsets []TrieNodePathSet
hashes []common.Hash
paths []string
)
for _, x := range []string{
"0x9",
@@ -67,15 +61,14 @@ func TestRequestSorting(t *testing.T) {
"0x01234567890123456789012345678901012345678901234567890123456789010",
"0x01234567890123456789012345678901012345678901234567890123456789011",
} {
sp, _, hash := f(x)
hashes = append(hashes, hash)
paths = append(paths, sp)
paths = append(paths, f(x))
hashes = append(hashes, common.Hash{})
}
_, paths, pathsets = sortByAccountPath(hashes, paths)
_, _, syncPaths, pathsets := sortByAccountPath(paths, hashes)
{
var b = new(bytes.Buffer)
for i := 0; i < len(paths); i++ {
fmt.Fprintf(b, "\n%d. paths %x", i, paths[i])
for i := 0; i < len(syncPaths); i++ {
fmt.Fprintf(b, "\n%d. paths %x", i, syncPaths[i])
}
want := `
0. paths [0099]
eth/protocols/snap/sync.go
+46 -41
View File
@@ -230,8 +230,8 @@ type trienodeHealRequest struct {
timeout *time.Timer // Timer to track delivery timeout
stale chan struct{} // Channel to signal the request was dropped
hashes []common.Hash // Trie node hashes to validate responses
paths []trie.SyncPath // Trie node paths requested for rescheduling
paths []string // Trie node paths for identifying trie node
hashes []common.Hash // Trie node hashes to validate responses
task *healTask // Task which this request is filling (only access fields through the runloop!!)
}
@@ -240,9 +240,9 @@ type trienodeHealRequest struct {
type trienodeHealResponse struct {
task *healTask // Task which this request is filling
hashes []common.Hash // Hashes of the trie nodes to avoid double hashing
paths []trie.SyncPath // Trie node paths requested for rescheduling missing ones
nodes [][]byte // Actual trie nodes to store into the database (nil = missing)
paths []string // Paths of the trie nodes
hashes []common.Hash // Hashes of the trie nodes to avoid double hashing
nodes [][]byte // Actual trie nodes to store into the database (nil = missing)
}
// bytecodeHealRequest tracks a pending bytecode request to ensure responses are to
@@ -321,8 +321,8 @@ type storageTask struct {
type healTask struct {
scheduler *trie.Sync // State trie sync scheduler defining the tasks
trieTasks map[common.Hash]trie.SyncPath // Set of trie node tasks currently queued for retrieval
codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval
trieTasks map[string]common.Hash // Set of trie node tasks currently queued for retrieval, indexed by node path
codeTasks map[common.Hash]struct{} // Set of byte code tasks currently queued for retrieval, indexed by code hash
}
// SyncProgress is a database entry to allow suspending and resuming a snapshot state
@@ -540,7 +540,7 @@ func (s *Syncer) Unregister(id string) error {
return nil
}
// Sync starts (or resumes a previous) sync cycle to iterate over an state trie
// Sync starts (or resumes a previous) sync cycle to iterate over a state trie
// with the given root and reconstruct the nodes based on the snapshot leaves.
// Previously downloaded segments will not be redownloaded of fixed, rather any
// errors will be healed after the leaves are fully accumulated.
@@ -551,7 +551,7 @@ func (s *Syncer) Sync(root common.Hash, cancel chan struct{}) error {
s.root = root
s.healer = &healTask{
scheduler: state.NewStateSync(root, s.db, s.onHealState),
trieTasks: make(map[common.Hash]trie.SyncPath),
trieTasks: make(map[string]common.Hash),
codeTasks: make(map[common.Hash]struct{}),
}
s.statelessPeers = make(map[string]struct{})
@@ -743,7 +743,7 @@ func (s *Syncer) loadSyncStatus() {
return
}
}
// Either we've failed to decode the previus state, or there was none.
// Either we've failed to decode the previous state, or there was none.
// Start a fresh sync by chunking up the account range and scheduling
// them for retrieval.
s.tasks = nil
@@ -1280,9 +1280,9 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai
want = maxTrieRequestCount + maxCodeRequestCount
)
if have < want {
nodes, paths, codes := s.healer.scheduler.Missing(want - have)
for i, hash := range nodes {
s.healer.trieTasks[hash] = paths[i]
paths, hashes, codes := s.healer.scheduler.Missing(want - have)
for i, path := range paths {
s.healer.trieTasks[path] = hashes[i]
}
for _, hash := range codes {
s.healer.codeTasks[hash] = struct{}{}
@@ -1323,21 +1323,20 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai
}
var (
hashes = make([]common.Hash, 0, cap)
paths = make([]trie.SyncPath, 0, cap)
paths = make([]string, 0, cap)
pathsets = make([]TrieNodePathSet, 0, cap)
)
for hash, pathset := range s.healer.trieTasks {
delete(s.healer.trieTasks, hash)
for path, hash := range s.healer.trieTasks {
delete(s.healer.trieTasks, path)
paths = append(paths, path)
hashes = append(hashes, hash)
paths = append(paths, pathset)
if len(hashes) >= cap {
if len(paths) >= cap {
break
}
}
// Group requests by account hash
hashes, paths, pathsets = sortByAccountPath(hashes, paths)
paths, hashes, _, pathsets = sortByAccountPath(paths, hashes)
req := &trienodeHealRequest{
peer: idle,
id: reqid,
@@ -1346,8 +1345,8 @@ func (s *Syncer) assignTrienodeHealTasks(success chan *trienodeHealResponse, fai
revert: fail,
cancel: cancel,
stale: make(chan struct{}),
hashes: hashes,
paths: paths,
hashes: hashes,
task: s.healer,
}
req.timeout = time.AfterFunc(s.rates.TargetTimeout(), func() {
@@ -1405,9 +1404,9 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai
want = maxTrieRequestCount + maxCodeRequestCount
)
if have < want {
nodes, paths, codes := s.healer.scheduler.Missing(want - have)
for i, hash := range nodes {
s.healer.trieTasks[hash] = paths[i]
paths, hashes, codes := s.healer.scheduler.Missing(want - have)
for i, path := range paths {
s.healer.trieTasks[path] = hashes[i]
}
for _, hash := range codes {
s.healer.codeTasks[hash] = struct{}{}
@@ -1703,10 +1702,10 @@ func (s *Syncer) revertTrienodeHealRequest(req *trienodeHealRequest) {
s.lock.Unlock()
// If there's a timeout timer still running, abort it and mark the trie node
// retrievals as not-pending, ready for resheduling
// retrievals as not-pending, ready for rescheduling
req.timeout.Stop()
for i, hash := range req.hashes {
req.task.trieTasks[hash] = req.paths[i]
for i, path := range req.paths {
req.task.trieTasks[path] = req.hashes[i]
}
}
@@ -2096,14 +2095,14 @@ func (s *Syncer) processTrienodeHealResponse(res *trienodeHealResponse) {
// If the trie node was not delivered, reschedule it
if node == nil {
res.task.trieTasks[hash] = res.paths[i]
res.task.trieTasks[res.paths[i]] = res.hashes[i]
continue
}
// Push the trie node into the state syncer
s.trienodeHealSynced++
s.trienodeHealBytes += common.StorageSize(len(node))
err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node})
err := s.healer.scheduler.ProcessNode(trie.NodeSyncResult{Path: res.paths[i], Data: node})
switch err {
case nil:
case trie.ErrAlreadyProcessed:
@@ -2139,7 +2138,7 @@ func (s *Syncer) processBytecodeHealResponse(res *bytecodeHealResponse) {
s.bytecodeHealSynced++
s.bytecodeHealBytes += common.StorageSize(len(node))
err := s.healer.scheduler.Process(trie.SyncResult{Hash: hash, Data: node})
err := s.healer.scheduler.ProcessCode(trie.CodeSyncResult{Hash: hash, Data: node})
switch err {
case nil:
case trie.ErrAlreadyProcessed:
@@ -2666,9 +2665,9 @@ func (s *Syncer) OnTrieNodes(peer SyncPeer, id uint64, trienodes [][]byte) error
}
// Response validated, send it to the scheduler for filling
response := &trienodeHealResponse{
paths: req.paths,
task: req.task,
hashes: req.hashes,
paths: req.paths,
nodes: nodes,
}
select {
@@ -2913,8 +2912,9 @@ func (s *capacitySort) Swap(i, j int) {
// healRequestSort implements the Sort interface, allowing sorting trienode
// heal requests, which is a prerequisite for merging storage-requests.
type healRequestSort struct {
hashes []common.Hash
paths []trie.SyncPath
paths []string
hashes []common.Hash
syncPaths []trie.SyncPath
}
func (t *healRequestSort) Len() int {
@@ -2922,8 +2922,8 @@ func (t *healRequestSort) Len() int {
}
func (t *healRequestSort) Less(i, j int) bool {
a := t.paths[i]
b := t.paths[j]
a := t.syncPaths[i]
b := t.syncPaths[j]
switch bytes.Compare(a[0], b[0]) {
case -1:
return true
@@ -2944,8 +2944,9 @@ func (t *healRequestSort) Less(i, j int) bool {
}
func (t *healRequestSort) Swap(i, j int) {
t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i]
t.paths[i], t.paths[j] = t.paths[j], t.paths[i]
t.hashes[i], t.hashes[j] = t.hashes[j], t.hashes[i]
t.syncPaths[i], t.syncPaths[j] = t.syncPaths[j], t.syncPaths[i]
}
// Merge merges the pathsets, so that several storage requests concerning the
@@ -2953,7 +2954,7 @@ func (t *healRequestSort) Swap(i, j int) {
// OBS: This operation is moot if t has not first been sorted.
func (t *healRequestSort) Merge() []TrieNodePathSet {
var result []TrieNodePathSet
for _, path := range t.paths {
for _, path := range t.syncPaths {
pathset := TrieNodePathSet([][]byte(path))
if len(path) == 1 {
// It's an account reference.
@@ -2962,7 +2963,7 @@ func (t *healRequestSort) Merge() []TrieNodePathSet {
// It's a storage reference.
end := len(result) - 1
if len(result) == 0 || !bytes.Equal(pathset[0], result[end][0]) {
// The account doesn't doesn't match last, create a new entry.
// The account doesn't match last, create a new entry.
result = append(result, pathset)
} else {
// It's the same account as the previous one, add to the storage
@@ -2976,9 +2977,13 @@ func (t *healRequestSort) Merge() []TrieNodePathSet {
// sortByAccountPath takes hashes and paths, and sorts them. After that, it generates
// the TrieNodePaths and merges paths which belongs to the same account path.
func sortByAccountPath(hashes []common.Hash, paths []trie.SyncPath) ([]common.Hash, []trie.SyncPath, []TrieNodePathSet) {
n := &healRequestSort{hashes, paths}
func sortByAccountPath(paths []string, hashes []common.Hash) ([]string, []common.Hash, []trie.SyncPath, []TrieNodePathSet) {
var syncPaths []trie.SyncPath
for _, path := range paths {
syncPaths = append(syncPaths, trie.NewSyncPath([]byte(path)))
}
n := &healRequestSort{paths, hashes, syncPaths}
sort.Sort(n)
pathsets := n.Merge()
return n.hashes, n.paths, pathsets
return n.paths, n.hashes, n.syncPaths, pathsets
}