cerc-io/lotus
10
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

don't delete in one giant batch, use smaller chunks of batchSize

Browse Source
This commit is contained in:
vyzo 2021-03-02 20:20:07 +02:00
parent 70ebb2ad8d
commit d2d0980532
1 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
blockstore/splitstore/splitstore.go
View File

@ -611,7 +611,9 @@ func (s *SplitStore) compactSimple(curTs *types.TipSet) {
panic(err) panic(err)
} }
s.coldPurgeSize = coldCnt if coldCnt > 0 {
s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
}
log.Infow("collection done", "took", time.Since(startCollect)) log.Infow("collection done", "took", time.Since(startCollect))
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt) log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
@ -701,9 +703,24 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
} }
func (s *SplitStore) purgeBlocks(cids []cid.Cid) error { func (s *SplitStore) purgeBlocks(cids []cid.Cid) error {
err := s.hot.DeleteMany(cids) if len(cids) == 0 {
if err != nil { return nil
return xerrors.Errorf("error deleting batch from hotstore: %e", err) }
// don't delete one giant batch of 7M objects, but rather do smaller batches
done := false
for i := 0; done; i++ {
start := i * batchSize
end := start + batchSize
if end >= len(cids) {
end = len(cids)
done = true
}
err := s.hot.DeleteMany(cids[start:end])
if err != nil {
return xerrors.Errorf("error deleting batch from hotstore: %e", err)
}
} }
return nil return nil
@ -854,8 +871,12 @@ func (s *SplitStore) compactFull(curTs *types.TipSet) {
panic(err) panic(err)
} }
s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit if coldCnt > 0 {
s.deadPurgeSize = deadCnt + deadCnt>>2 // overestimate a bit s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
}
if deadCnt > 0 {
s.deadPurgeSize = deadCnt + deadCnt>>2 // overestimate a bit
}
log.Infow("collection done", "took", time.Since(startCollect)) log.Infow("collection done", "took", time.Since(startCollect))
log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "dead", deadCnt) log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "dead", deadCnt)