keep track of the compaction serial (index)

it is useful so that:
- we only do slow (but very effective) moving gc every 10 compactions
- we can detect a splitstore v0 upgrade and re-warm up

blockstore/splitstore/splitstore.go

@@ -72,6 +72,9 @@ var (
 	// this is first computed at warmup and updated in every compaction
 	markSetSizeKey = dstore.NewKey("/splitstore/markSetSize")
 
+	// compactionIndexKey stores the compaction index (serial number)
+	compactionIndexKey = dstore.NewKey("/splitstore/compactionIndex")
+
 	log = logging.Logger("splitstore")
 
 	// used to signal end of walk
@@ -140,6 +143,8 @@ type SplitStore struct {
 	markSetEnv  MarkSetEnv
 	markSetSize int64
 
+	compactionIndex int64
+
 	ctx    context.Context
 	cancel func()
 
@@ -530,6 +535,17 @@ func (s *SplitStore) Start(chain ChainAccessor) error {
 		return xerrors.Errorf("error loading mark set size: %w", err)
 	}
 
+	// load compactionIndex from metadata ds to provide a hint as to when to perform moving gc
+	bs, err = s.ds.Get(compactionIndexKey)
+	switch err {
+	case nil:
+		s.compactionIndex = bytesToInt64(bs)
+
+	case dstore.ErrNotFound:
+	default:
+		return xerrors.Errorf("error loading compaction index: %w", err)
+	}
+
 	log.Infow("starting splitstore", "baseEpoch", s.baseEpoch, "warmupEpoch", s.warmupEpoch)
 
 	// watch the chain
@@ -977,7 +993,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	currentEpoch := curTs.Height()
 	boundaryEpoch := currentEpoch - CompactionBoundary
 
-	log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch)
+	log.Infow("running compaction", "currentEpoch", currentEpoch, "baseEpoch", s.baseEpoch, "boundaryEpoch", boundaryEpoch, "compactionIndex", s.compactionIndex)
 
 	markSet, err := s.markSetEnv.Create("live", s.markSetSize)
 	if err != nil {
@@ -994,7 +1010,7 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 	s.beginTxnMarking(markSet)
 
 	// 1. mark reachable objects by walking the chain from the current epoch; we keep state roots
-	//   and messages until the boundary epoch.
+	//   and messages until the boundary epoch.nn
 	log.Info("marking reachable objects")
 	startMark := time.Now()
 
@@ -1147,6 +1163,12 @@ func (s *SplitStore) doCompact(curTs *types.TipSet) error {
 		return xerrors.Errorf("error saving mark set size: %w", err)
 	}
 
+	s.compactionIndex++
+	err = s.ds.Put(compactionIndexKey, int64ToBytes(s.compactionIndex))
+	if err != nil {
+		return xerrors.Errorf("error saving compaction index: %w", err)
+	}
+
 	return nil
 }
 

blockstore/splitstore/splitstore_gc.go

@@ -7,6 +7,11 @@ import (
 )
 
 func (s *SplitStore) gcHotstore() {
+	// we only perform moving gc every 10 compactions as it can take a while
+	if s.compactionIndex%10 != 0 {
+		goto online_gc
+	}
+
 	// check if the hotstore is movable; if so, move it.
 	if mover, ok := s.hot.(bstore.BlockstoreMover); ok {
 		log.Info("moving hotstore")
@@ -14,13 +19,15 @@ func (s *SplitStore) gcHotstore() {
 		err := mover.MoveTo("", nil)
 		if err != nil {
 			log.Warnf("error moving hotstore: %s", err)
+			// try online gc
+			goto online_gc
+		}
+
+		log.Infow("moving hotstore done", "took", time.Since(startMove))
 		return
 	}
 
-		log.Infof("moving hotstore done", "took", time.Since(startMove))
-		return
-	}
-
+online_gc:
 	// check if the hotstore supports online GC; if so, GC it.
 	if gc, ok := s.hot.(bstore.BlockstoreGC); ok {
 		log.Info("garbage collecting hotstore")