From 642f0e47407d91230845ac56273d9542626b9b0d Mon Sep 17 00:00:00 2001 From: vyzo Date: Sun, 4 Jul 2021 06:21:04 +0300 Subject: [PATCH] deal with memory pressure, don't walk under the boundary --- blockstore/splitstore/splitstore.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index c4a554903..4b170f91f 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -177,7 +177,7 @@ func Open(path string, ds dstore.Datastore, hot, cold bstore.Blockstore, cfg *Co } // the markset env - markSetEnv, err := OpenMarkSetEnv(path, "mapts") + markSetEnv, err := OpenMarkSetEnv(path, "bolt") if err != nil { _ = tracker.Close() return nil, err @@ -964,7 +964,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { // create the transaction protect filter s.txnLk.Lock() - s.txnProtect, err = s.txnEnv.Create("protected", s.markSetSize) + s.txnProtect, err = s.txnEnv.Create("protected", 0) if err != nil { s.txnLk.Unlock() return xerrors.Errorf("error creating transactional mark set: %w", err) @@ -988,10 +988,10 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { startCollect := time.Now() candidates := make(map[cid.Cid]struct{}, s.coldPurgeSize) - towalk := make([]cid.Cid, 0, count) + var towalk []cid.Cid // some stats for logging - var hotCnt, coldCnt, liveCnt int + var hotCnt, coldCnt, slackCnt, liveCnt int // 2.1 iterate through the tracking store and collect unreachable cold objects // for every hot object that is a dag and not in the markset, walk for links and @@ -1009,7 +1009,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { } // is the object still hot? - if writeEpoch > coldEpoch { + if writeEpoch >= boundaryEpoch { // yes, stay in the hotstore hotCnt++ @@ -1022,6 +1022,14 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { return nil } + // is the object in slack region? + if writeEpoch > coldEpoch { + // yes stay in the hotstore, but we wont walk you + slackCnt++ + + return nil + } + // it's cold, mark it as candidate for move candidates[c] = struct{}{} coldCnt++ @@ -1089,7 +1097,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error { cold = append(cold, c) } - log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "live", liveCnt) + log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "live", liveCnt, "slack", slackCnt) stats.Record(context.Background(), metrics.SplitstoreCompactionHot.M(int64(hotCnt))) stats.Record(context.Background(), metrics.SplitstoreCompactionCold.M(int64(coldCnt)))