cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

chainwatch: sync in batches

Browse Source
This commit is contained in:
Łukasz Magiera 2020-01-08 17:29:46 +01:00
parent 1b5e1c4753
commit 79028397ad
1 changed files with 164 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
cmd/lotus-chainwatch/sync.go
View File

@ -4,6 +4,7 @@ import (
"bytes" "bytes"
"container/list" "container/list"
"context" "context"
"math"
"sync" "sync"
"github.com/filecoin-project/go-address" "github.com/filecoin-project/go-address"
@ -16,6 +17,8 @@ import (
"github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/chain/types"
) )
const maxBatch = 10000
func runSyncer(ctx context.Context, api api.FullNode, st *storage) { func runSyncer(ctx context.Context, api api.FullNode, st *storage) {
notifs, err := api.ChainNotify(ctx) notifs, err := api.ChainNotify(ctx)
if err != nil { if err != nil {
@ -68,7 +71,7 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Infof("Getting headers / actors") log.Infof("Getting headers / actors")
toSync := map[cid.Cid]*types.BlockHeader{} allToSync := map[cid.Cid]*types.BlockHeader{}
toVisit := list.New() toVisit := list.New()
for _, header := range ts.Blocks() { for _, header := range ts.Blocks() {
@ -79,15 +82,15 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader) bh := toVisit.Remove(toVisit.Back()).(*types.BlockHeader)
_, has := hazlist[bh.Cid()] _, has := hazlist[bh.Cid()]
if _, seen := toSync[bh.Cid()]; seen || has { if _, seen := allToSync[bh.Cid()]; seen || has {
continue continue
} }
toSync[bh.Cid()] = bh allToSync[bh.Cid()] = bh
addresses[bh.Miner] = address.Undef addresses[bh.Miner] = address.Undef
if len(toSync)%500 == 10 { if len(allToSync)%500 == 10 {
log.Infof("todo: (%d) %s", len(toSync), bh.Cid()) log.Infof("todo: (%d) %s @%d", len(allToSync), bh.Cid(), bh.Height)
} }
if len(bh.Parents) == 0 { if len(bh.Parents) == 0 {
@ -105,6 +108,25 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
} }
} }
for len(allToSync) > 0 {
minH := uint64(math.MaxUint64)
for _, header := range allToSync {
if header.Height < minH {
minH = header.Height
}
}
toSync := map[cid.Cid]*types.BlockHeader{}
for c, header := range allToSync {
if header.Height < minH+maxBatch {
toSync[c] = header
}
}
for c := range toSync {
delete(allToSync, c)
}
log.Infof("Syncing %d blocks", len(toSync)) log.Infof("Syncing %d blocks", len(toSync))
paDone := 0 paDone := 0
@ -297,6 +319,8 @@ func syncHead(ctx context.Context, api api.FullNode, st *storage, ts *types.TipS
log.Error(err) log.Error(err)
return return
} }
log.Infof("Sync stage done")
}
log.Infof("Sync done") log.Infof("Sync done")
} }