From d2d0980532c9bb12043ef561b802ac6e69999ee5 Mon Sep 17 00:00:00 2001
From: vyzo <vyzo@hackzen.org>
Date: Tue, 2 Mar 2021 20:20:07 +0200
Subject: [PATCH] don't delete in one giant batch, use smaller chunks of
 batchSize

---
 blockstore/splitstore/splitstore.go | 33 +++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go
index 15d16afb7..0e4c3e150 100644
--- a/blockstore/splitstore/splitstore.go
+++ b/blockstore/splitstore/splitstore.go
@@ -611,7 +611,9 @@ func (s *SplitStore) compactSimple(curTs *types.TipSet) {
 		panic(err)
 	}
 
-	s.coldPurgeSize = coldCnt
+	if coldCnt > 0 {
+		s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
+	}
 
 	log.Infow("collection done", "took", time.Since(startCollect))
 	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt)
@@ -701,9 +703,24 @@ func (s *SplitStore) moveColdBlocks(cold []cid.Cid) error {
 }
 
 func (s *SplitStore) purgeBlocks(cids []cid.Cid) error {
-	err := s.hot.DeleteMany(cids)
-	if err != nil {
-		return xerrors.Errorf("error deleting batch from hotstore: %e", err)
+	if len(cids) == 0 {
+		return nil
+	}
+
+	// don't delete one giant batch of 7M objects, but rather do smaller batches
+	done := false
+	for i := 0; done; i++ {
+		start := i * batchSize
+		end := start + batchSize
+		if end >= len(cids) {
+			end = len(cids)
+			done = true
+		}
+
+		err := s.hot.DeleteMany(cids[start:end])
+		if err != nil {
+			return xerrors.Errorf("error deleting batch from hotstore: %e", err)
+		}
 	}
 
 	return nil
@@ -854,8 +871,12 @@ func (s *SplitStore) compactFull(curTs *types.TipSet) {
 		panic(err)
 	}
 
-	s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
-	s.deadPurgeSize = deadCnt + deadCnt>>2 // overestimate a bit
+	if coldCnt > 0 {
+		s.coldPurgeSize = coldCnt + coldCnt>>2 // overestimate a bit
+	}
+	if deadCnt > 0 {
+		s.deadPurgeSize = deadCnt + deadCnt>>2 // overestimate a bit
+	}
 
 	log.Infow("collection done", "took", time.Since(startCollect))
 	log.Infow("compaction stats", "hot", hotCnt, "cold", coldCnt, "dead", deadCnt)