cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

fix warmup by decoupling state from message receipt walk

Browse Source
This commit is contained in:
vyzo 2021-07-22 20:49:36 +03:00
parent 2d40089b7e
commit 2a68ae8dad
2 changed files with 16 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
blockstore/splitstore/splitstore_compact.go
View File

@ -436,7 +436,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
startMark := time.Now() startMark := time.Now()
var count int64 var count int64
err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, err = s.walkChain(curTs, boundaryEpoch, inclMsgsEpoch, boundaryEpoch,
func(c cid.Cid) error { func(c cid.Cid) error {
if isUnitaryObject(c) { if isUnitaryObject(c) {
return errStopWalk return errStopWalk
@ -631,7 +631,7 @@ func (s *SplitStore) endTxnProtect() {
s.txnMissing = nil s.txnMissing = nil
} }
func (s *SplitStore) walkChain(ts *types.TipSet, inclState abi.ChainEpoch, inclMsgs abi.ChainEpoch, func (s *SplitStore) walkChain(ts *types.TipSet, inclState, inclMsgs, inclMsgReceipts abi.ChainEpoch,
f func(cid.Cid) error) error { f func(cid.Cid) error) error {
visited := cid.NewSet() visited := cid.NewSet()
walked := cid.NewSet() walked := cid.NewSet()
@ -639,6 +639,8 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState abi.ChainEpoch, inclM
walkCnt := 0 walkCnt := 0
scanCnt := 0 scanCnt := 0
stopWalk := func(_ cid.Cid) error { return errStopWalk }
walkBlock := func(c cid.Cid) error { walkBlock := func(c cid.Cid) error {
if !visited.Visit(c) { if !visited.Visit(c) {
return nil return nil
@ -664,7 +666,6 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState abi.ChainEpoch, inclM
if inclMsgs < inclState { if inclMsgs < inclState {
// we need to use walkObjectIncomplete here, as messages may be missing early on if we // we need to use walkObjectIncomplete here, as messages may be missing early on if we
// synced from snapshot and have a long HotStoreMessageRetentionPolicy. // synced from snapshot and have a long HotStoreMessageRetentionPolicy.
stopWalk := func(_ cid.Cid) error { return errStopWalk }
if err := s.walkObjectIncomplete(hdr.Messages, walked, f, stopWalk); err != nil { if err := s.walkObjectIncomplete(hdr.Messages, walked, f, stopWalk); err != nil {
return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err) return xerrors.Errorf("error walking messages (cid: %s): %w", hdr.Messages, err)
} }
@ -675,14 +676,22 @@ func (s *SplitStore) walkChain(ts *types.TipSet, inclState abi.ChainEpoch, inclM
} }
} }
// state and message receipts is only retained if within the inclState boundary if hdr.Height >= inclMsgReceipts && hdr.Height > 0 {
if hdr.Height >= inclState || hdr.Height == 0 { if inclMsgReceipts < inclState {
if hdr.Height > 0 { // we need to use walkObjectIncomplete here, as message receipts may be missing early on if we
// synced from snapshot
if err := s.walkObjectIncomplete(hdr.ParentMessageReceipts, walked, f, stopWalk); err != nil {
return xerrors.Errorf("error walking messages receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
}
} else {
if err := s.walkObject(hdr.ParentMessageReceipts, walked, f); err != nil { if err := s.walkObject(hdr.ParentMessageReceipts, walked, f); err != nil {
return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err) return xerrors.Errorf("error walking message receipts (cid: %s): %w", hdr.ParentMessageReceipts, err)
} }
} }
}
// state is only retained if within the inclState boundary, with the exception of genesis
if hdr.Height >= inclState || hdr.Height == 0 {
if err := s.walkObject(hdr.ParentStateRoot, walked, f); err != nil { if err := s.walkObject(hdr.ParentStateRoot, walked, f); err != nil {
return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err) return xerrors.Errorf("error walking state root (cid: %s): %w", hdr.ParentStateRoot, err)
} }

2
blockstore/splitstore/splitstore_warmup.go
View File

@ -48,7 +48,7 @@ func (s *SplitStore) doWarmup(curTs *types.TipSet) error {
count := int64(0) count := int64(0)
xcount := int64(0) xcount := int64(0)
missing := int64(0) missing := int64(0)
err := s.walkChain(curTs, epoch, epoch+1, // we don't load messages in warmup err := s.walkChain(curTs, epoch, epoch+1, epoch+1, // we don't load messages/receipts in warmup
func(c cid.Cid) error { func(c cid.Cid) error {
if isUnitaryObject(c) { if isUnitaryObject(c) {
return errStopWalk return errStopWalk