From 54a6968327190593e931fedae0530c7ab221bbdb Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 12:11:49 +0300 Subject: [PATCH 01/15] add badger-backed markset implementation --- blockstore/splitstore/markset.go | 2 + blockstore/splitstore/markset_badger.go | 183 ++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 blockstore/splitstore/markset_badger.go diff --git a/blockstore/splitstore/markset.go b/blockstore/splitstore/markset.go index a644e7279..458ea8beb 100644 --- a/blockstore/splitstore/markset.go +++ b/blockstore/splitstore/markset.go @@ -32,6 +32,8 @@ func OpenMarkSetEnv(path string, mtype string) (MarkSetEnv, error) { return NewBloomMarkSetEnv() case "map": return NewMapMarkSetEnv() + case "badger": + return NewBadgerMarkSetEnv(path) default: return nil, xerrors.Errorf("unknown mark set type %s", mtype) } diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go new file mode 100644 index 000000000..41d2601c3 --- /dev/null +++ b/blockstore/splitstore/markset_badger.go @@ -0,0 +1,183 @@ +package splitstore + +import ( + "os" + "path/filepath" + "sync" + + "golang.org/x/xerrors" + + "github.com/dgraph-io/badger/v2" + "go.uber.org/zap" + + cid "github.com/ipfs/go-cid" +) + +type BadgerMarkSetEnv struct { + path string +} + +var _ MarkSetEnv = (*BadgerMarkSetEnv)(nil) + +type BadgerMarkSet struct { + mx sync.RWMutex + pend map[string]struct{} + + db *badger.DB + path string +} + +var _ MarkSet = (*BadgerMarkSet)(nil) + +const badgerMarkSetBatchSize = 65536 + +func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) { + msPath := filepath.Join(path, "markset.badger") + err := os.MkdirAll(msPath, 0755) //nolint:gosec + if err != nil { + return nil, xerrors.Errorf("error creating markset directory: %w", err) + } + + return &BadgerMarkSetEnv{path: msPath}, nil +} + +func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) { + path := filepath.Join(e.path, name) + + // clean up first + err := os.RemoveAll(path) + if err != nil { + return nil, xerrors.Errorf("error clearing markset directory: %w", err) + } + + err = os.MkdirAll(path, 0755) //nolint:gosec + if err != nil { + return nil, xerrors.Errorf("error creating markset directory: %w", err) + } + + opts := badger.DefaultOptions(path) + opts.SyncWrites = false + opts.Logger = &badgerLogger{ + SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), + skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), + } + + db, err := badger.Open(opts) + if err != nil { + return nil, xerrors.Errorf("error creating badger markset: %w", err) + } + + return &BadgerMarkSet{ + pend: make(map[string]struct{}), + db: db, + path: path, + }, nil +} + +func (e *BadgerMarkSetEnv) Close() error { + return os.RemoveAll(e.path) +} + +func (s *BadgerMarkSet) Mark(c cid.Cid) error { + s.mx.Lock() + defer s.mx.Unlock() + + if s.pend == nil { + return errMarkSetClosed + } + + s.pend[string(c.Hash())] = struct{}{} + + if len(s.pend) < badgerMarkSetBatchSize { + return nil + } + + pend := s.pend + s.pend = make(map[string]struct{}) + + empty := []byte{} // not nil + + batch := s.db.NewWriteBatch() + defer batch.Cancel() + + for k := range pend { + if err := batch.Set([]byte(k), empty); err != nil { + return err + } + } + + err := batch.Flush() + if err != nil { + return xerrors.Errorf("error flushing batch to badger markset: %w", err) + } + + return nil +} + +func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) { + s.mx.RLock() + defer s.mx.RUnlock() + + if s.pend == nil { + return false, errMarkSetClosed + } + + key := c.Hash() + _, ok := s.pend[string(key)] + if ok { + return true, nil + } + + err := s.db.View(func(txn *badger.Txn) error { + _, err := txn.Get(key) + return err + }) + + switch err { + case nil: + return true, nil + + case badger.ErrKeyNotFound: + return false, nil + + default: + return false, xerrors.Errorf("error checking badger markset: %w", err) + } +} + +func (s *BadgerMarkSet) Close() error { + s.mx.Lock() + defer s.mx.Unlock() + + if s.pend == nil { + return nil + } + + s.pend = nil + db := s.db + s.db = nil + + err := db.Close() + if err != nil { + return xerrors.Errorf("error closing badger markset: %w", err) + } + + err = os.RemoveAll(s.path) + if err != nil { + return xerrors.Errorf("error deleting badger markset: %w", err) + } + + return nil +} + +func (s *BadgerMarkSet) SetConcurrent() {} + +// badger logging through go-log +type badgerLogger struct { + *zap.SugaredLogger + skip2 *zap.SugaredLogger +} + +func (b *badgerLogger) Warningf(format string, args ...interface{}) { + b.skip2.Warnf(format, args...) +} From 2c26abc551693d40bbe2e8ee24df0c3cf1dd4f18 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 12:16:44 +0300 Subject: [PATCH 02/15] add badger markset test --- blockstore/splitstore/markset_badger.go | 2 +- blockstore/splitstore/markset_test.go | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 41d2601c3..fd979e2e8 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -29,7 +29,7 @@ type BadgerMarkSet struct { var _ MarkSet = (*BadgerMarkSet)(nil) -const badgerMarkSetBatchSize = 65536 +var badgerMarkSetBatchSize = 65536 func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) { msPath := filepath.Join(path, "markset.badger") diff --git a/blockstore/splitstore/markset_test.go b/blockstore/splitstore/markset_test.go index d5c01e220..38519949a 100644 --- a/blockstore/splitstore/markset_test.go +++ b/blockstore/splitstore/markset_test.go @@ -16,6 +16,15 @@ func TestBloomMarkSet(t *testing.T) { testMarkSet(t, "bloom") } +func TestBadgerMarkSet(t *testing.T) { + bs := badgerMarkSetBatchSize + badgerMarkSetBatchSize = 1 + t.Cleanup(func() { + badgerMarkSetBatchSize = bs + }) + testMarkSet(t, "badger") +} + func testMarkSet(t *testing.T, lsType string) { t.Helper() From be9530b66e886c098ff6b8a5626b0686859fab8b Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 12:27:59 +0300 Subject: [PATCH 03/15] finetune badger options --- blockstore/splitstore/markset_badger.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index fd979e2e8..5adaadd2e 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -8,6 +8,7 @@ import ( "golang.org/x/xerrors" "github.com/dgraph-io/badger/v2" + "github.com/dgraph-io/badger/v2/options" "go.uber.org/zap" cid "github.com/ipfs/go-cid" @@ -57,6 +58,7 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) opts := badger.DefaultOptions(path) opts.SyncWrites = false + opts.Compression = options.None opts.Logger = &badgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), @@ -178,6 +180,6 @@ type badgerLogger struct { skip2 *zap.SugaredLogger } -func (b *badgerLogger) Warningf(format string, args ...interface{}) { - b.skip2.Warnf(format, args...) -} +func (b *badgerLogger) Warningf(format string, args ...interface{}) {} +func (b *badgerLogger) Infof(format string, args ...interface{}) {} +func (b *badgerLogger) Debugf(format string, args ...interface{}) {} From 12c3432b8dca4486c51203171e2a40d2bed3d5f8 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 13:00:54 +0300 Subject: [PATCH 04/15] document the "badger" markset type option --- blockstore/splitstore/splitstore.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blockstore/splitstore/splitstore.go b/blockstore/splitstore/splitstore.go index 821ebb2b6..b401d657e 100644 --- a/blockstore/splitstore/splitstore.go +++ b/blockstore/splitstore/splitstore.go @@ -62,8 +62,11 @@ func init() { type Config struct { // MarkSetType is the type of mark set to use. // - // Only current sane value is "map", but we may add an option for a disk-backed - // markset for memory-constrained situations. + // The default value is "map", which uses an in-memory map-backed markset. + // If you are constrained in memory (i.e. compaction runs out of memory), you + // can use "badger", which will use a disk-backed markset using badger. + // Note that compaction will take quite a bit longer when using the "badger" option, + // but that shouldn't really matter (as long as it is under 7.5hrs). MarkSetType string // DiscardColdBlocks indicates whether to skip moving cold blocks to the coldstore. From f2b7c3e6f2e41f33221802438e3cc0590151134d Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 13:47:40 +0300 Subject: [PATCH 05/15] reduce scope of exclusive lock in badger markset --- blockstore/splitstore/markset_badger.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 5adaadd2e..388478a3e 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -82,24 +82,27 @@ func (e *BadgerMarkSetEnv) Close() error { func (s *BadgerMarkSet) Mark(c cid.Cid) error { s.mx.Lock() - defer s.mx.Unlock() if s.pend == nil { + s.mx.Unlock() return errMarkSetClosed } s.pend[string(c.Hash())] = struct{}{} if len(s.pend) < badgerMarkSetBatchSize { + s.mx.Unlock() return nil } pend := s.pend s.pend = make(map[string]struct{}) + db := s.db + s.mx.Unlock() empty := []byte{} // not nil - batch := s.db.NewWriteBatch() + batch := db.NewWriteBatch() defer batch.Cancel() for k := range pend { From 2891a31c997257b8fb2a1006be8a1aadcff3e32f Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 13:54:50 +0300 Subject: [PATCH 06/15] make badger markset concurrent close safe --- blockstore/splitstore/markset_badger.go | 30 ++++++++++++++++++++----- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 388478a3e..7063a5317 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -21,8 +21,10 @@ type BadgerMarkSetEnv struct { var _ MarkSetEnv = (*BadgerMarkSetEnv)(nil) type BadgerMarkSet struct { - mx sync.RWMutex - pend map[string]struct{} + mx sync.RWMutex + cond sync.Cond + pend map[string]struct{} + writers int db *badger.DB path string @@ -69,11 +71,14 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) return nil, xerrors.Errorf("error creating badger markset: %w", err) } - return &BadgerMarkSet{ + ms := &BadgerMarkSet{ pend: make(map[string]struct{}), db: db, path: path, - }, nil + } + ms.cond.L = &ms.mx + + return ms, nil } func (e *BadgerMarkSetEnv) Close() error { @@ -97,12 +102,21 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error { pend := s.pend s.pend = make(map[string]struct{}) - db := s.db + s.writers++ s.mx.Unlock() + defer func() { + s.mx.Lock() + s.writers-- + if s.writers == 0 { + s.cond.Broadcast() + } + s.mx.Unlock() + }() + empty := []byte{} // not nil - batch := db.NewWriteBatch() + batch := s.db.NewWriteBatch() defer batch.Cancel() for k := range pend { @@ -158,6 +172,10 @@ func (s *BadgerMarkSet) Close() error { return nil } + for s.writers > 0 { + s.cond.Wait() + } + s.pend = nil db := s.db s.db = nil From f75d982c76e66ed90de9f8893a7dc7263218f6bf Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 14:31:39 +0300 Subject: [PATCH 07/15] remove early occurs check from trackTxnRef this happens inline, and it might block when using the badger markset --- blockstore/splitstore/splitstore_compact.go | 26 --------------------- 1 file changed, 26 deletions(-) diff --git a/blockstore/splitstore/splitstore_compact.go b/blockstore/splitstore/splitstore_compact.go index b0ba30100..b95459ea5 100644 --- a/blockstore/splitstore/splitstore_compact.go +++ b/blockstore/splitstore/splitstore_compact.go @@ -184,16 +184,6 @@ func (s *SplitStore) trackTxnRef(c cid.Cid) { return } - if s.txnProtect != nil { - mark, err := s.txnProtect.Has(c) - if err != nil { - log.Warnf("error checking markset: %s", err) - // track it anyways - } else if mark { - return - } - } - s.txnRefsMx.Lock() s.txnRefs[c] = struct{}{} s.txnRefsMx.Unlock() @@ -209,27 +199,11 @@ func (s *SplitStore) trackTxnRefMany(cids []cid.Cid) { s.txnRefsMx.Lock() defer s.txnRefsMx.Unlock() - quiet := false for _, c := range cids { if isUnitaryObject(c) { continue } - if s.txnProtect != nil { - mark, err := s.txnProtect.Has(c) - if err != nil { - if !quiet { - quiet = true - log.Warnf("error checking markset: %s", err) - } - // track it anyways - } - - if mark { - continue - } - } - s.txnRefs[c] = struct{}{} } From f492f8d450966c6eab3bba947e2e33a399f5dd5d Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 14:43:00 +0300 Subject: [PATCH 08/15] track in flight writes in badger markste, as they happen outside the lock --- blockstore/splitstore/markset_badger.go | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 7063a5317..c77099672 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -24,7 +24,9 @@ type BadgerMarkSet struct { mx sync.RWMutex cond sync.Cond pend map[string]struct{} + writing map[int]map[string]struct{} writers int + seqno int db *badger.DB path string @@ -72,9 +74,10 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) } ms := &BadgerMarkSet{ - pend: make(map[string]struct{}), - db: db, - path: path, + pend: make(map[string]struct{}), + writing: make(map[int]map[string]struct{}), + db: db, + path: path, } ms.cond.L = &ms.mx @@ -101,12 +104,17 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error { } pend := s.pend + seqno := s.seqno + s.seqno++ + s.writing[seqno] = pend s.pend = make(map[string]struct{}) s.writers++ s.mx.Unlock() defer func() { s.mx.Lock() + + delete(s.writing, seqno) s.writers-- if s.writers == 0 { s.cond.Broadcast() @@ -142,11 +150,19 @@ func (s *BadgerMarkSet) Has(c cid.Cid) (bool, error) { } key := c.Hash() - _, ok := s.pend[string(key)] + pendKey := string(key) + _, ok := s.pend[pendKey] if ok { return true, nil } + for _, wr := range s.writing { + _, ok := wr[pendKey] + if ok { + return true, nil + } + } + err := s.db.View(func(txn *badger.Txn) error { _, err := txn.Get(key) return err From 4df60733824696d92833e63abe948266de1c4e19 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 22 Jul 2021 15:24:30 +0300 Subject: [PATCH 09/15] some tweaks in badger markset --- blockstore/splitstore/markset_badger.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index c77099672..d2303dae2 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -34,7 +34,7 @@ type BadgerMarkSet struct { var _ MarkSet = (*BadgerMarkSet)(nil) -var badgerMarkSetBatchSize = 65536 +var badgerMarkSetBatchSize = 16384 func NewBadgerMarkSetEnv(path string) (MarkSetEnv, error) { msPath := filepath.Join(path, "markset.badger") @@ -113,13 +113,13 @@ func (s *BadgerMarkSet) Mark(c cid.Cid) error { defer func() { s.mx.Lock() + defer s.mx.Unlock() delete(s.writing, seqno) s.writers-- if s.writers == 0 { s.cond.Broadcast() } - s.mx.Unlock() }() empty := []byte{} // not nil From a8c0fd16543ca899eb99bae191fc3361a99c89ed Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 10:07:12 +0300 Subject: [PATCH 10/15] tweak badger options --- blockstore/splitstore/markset_badger.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index d2303dae2..6e0e33821 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -62,7 +62,10 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) opts := badger.DefaultOptions(path) opts.SyncWrites = false + opts.CompactL0OnClose = false opts.Compression = options.None + opts.TableLoadingMode = options.FileIO + opts.ValueLogLoadingMode = options.FileIO opts.Logger = &badgerLogger{ SugaredLogger: log.Desugar().WithOptions(zap.AddCallerSkip(1)).Sugar(), skip2: log.Desugar().WithOptions(zap.AddCallerSkip(2)).Sugar(), From dec5e13c8593851ab11066ad502500317e5c7155 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 11:37:41 +0300 Subject: [PATCH 11/15] update README with configuration options --- blockstore/splitstore/README.md | 33 +++++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/blockstore/splitstore/README.md b/blockstore/splitstore/README.md index 1c6569a34..48e9463db 100644 --- a/blockstore/splitstore/README.md +++ b/blockstore/splitstore/README.md @@ -27,9 +27,34 @@ If you intend to use the discard coldstore, your also need to add the following: ColdStoreType = "discard" ``` In general you _should not_ have to use the discard store, unless you -are running a network booster or have very constrained hardware with -not enough disk space to maintain a coldstore, even with garbage -collection. +are running a network assistive node (like a bootstrapper or booster) +or have very constrained hardware with not enough disk space to +maintain a coldstore, even with garbage collection. It is also appropriate +for small nodes that are simply watching the chain. + +## Configuration Options + +These are options in the `[Chainstore.Splitstore]` section of the configuration: + +- `HotStoreType` -- specifies the type of hotstore to use. + The only currently supported option is `"badger"`. +- `ColdStoreType` -- specifies the type of coldstore to use. + The default value is `"universal"`, which will use the initial monolith blockstore + as the coldstore. + The other possible value is `"discard"`, as outlined above, which is specialized for + running without a coldstore. Note that the discard store wraps the initial monolith + blockstore and discards writes; this is necessary to support syncing from a snapshot. +- `MarkSetType` -- specifies the type of markset to use during compaction. + The default value is `"map"`, which will use an in-memory map; if you are limited + in memory (or indeed see compaction run out of memory), you can also specify + `"badger"` which will use an disk backed markset, using badger. This will use + much less memory, but will also make compaction slower. +- `HotStoreMessageRetention` -- specifies how many finalities, beyond the 4 + finalities maintained by default, to maintain messages and message receipts in the + hotstore. This is useful for assistive nodes that want to support syncing for other + nodes beyond 4 finalities, while running with the discard coldstore option. + It is also useful for miners who accept deals and need to lookback messages beyond + the 4 finalities, which would otherwise hit the coldstore. ## Operation @@ -67,6 +92,6 @@ Compaction works transactionally with the following algorithm: - We delete in small batches taking a lock; each batch is checked again for marks, from the concurrent transactional mark, so as to never delete anything live - We then end the transaction and compact/gc the hotstore. -## Coldstore Garbage Collection +## Garbage Collection TBD -- see [#6577](https://github.com/filecoin-project/lotus/issues/6577) From 9d5cf4177b6a4ce9253c115864cd966caf59c1d4 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 11:38:01 +0300 Subject: [PATCH 12/15] add splitstore compaction test with badger markset --- blockstore/splitstore/splitstore_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index 26e5c3cc0..1d285b5a5 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -210,6 +210,10 @@ func TestSplitStoreCompaction(t *testing.T) { testSplitStore(t, &Config{MarkSetType: "map"}) } +func TestSplitStoreCompactionWithBadger(t *testing.T) { + testSplitStore(t, &Config{MarkSetType: "badger"}) +} + type mockChain struct { t testing.TB From 9faa3d5fd7c6fe39cc5a770ab247f698e0256083 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 12:07:38 +0300 Subject: [PATCH 13/15] adjust compaction test with badger to hit the db --- blockstore/splitstore/splitstore_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/blockstore/splitstore/splitstore_test.go b/blockstore/splitstore/splitstore_test.go index 1d285b5a5..b945eb90b 100644 --- a/blockstore/splitstore/splitstore_test.go +++ b/blockstore/splitstore/splitstore_test.go @@ -211,6 +211,11 @@ func TestSplitStoreCompaction(t *testing.T) { } func TestSplitStoreCompactionWithBadger(t *testing.T) { + bs := badgerMarkSetBatchSize + badgerMarkSetBatchSize = 1 + t.Cleanup(func() { + badgerMarkSetBatchSize = bs + }) testSplitStore(t, &Config{MarkSetType: "badger"}) } From 1480c224470cf9d9f0a4505a56d6145e45478d63 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 16:27:09 +0300 Subject: [PATCH 14/15] improve README --- blockstore/splitstore/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blockstore/splitstore/README.md b/blockstore/splitstore/README.md index 48e9463db..5b0df61d9 100644 --- a/blockstore/splitstore/README.md +++ b/blockstore/splitstore/README.md @@ -32,6 +32,9 @@ or have very constrained hardware with not enough disk space to maintain a coldstore, even with garbage collection. It is also appropriate for small nodes that are simply watching the chain. +*Warning:* Using the discard store for a general purpose node is discouraged, unless +you really know what you are doing. Use it at your own risk. + ## Configuration Options These are options in the `[Chainstore.Splitstore]` section of the configuration: @@ -45,6 +48,7 @@ These are options in the `[Chainstore.Splitstore]` section of the configuration: running without a coldstore. Note that the discard store wraps the initial monolith blockstore and discards writes; this is necessary to support syncing from a snapshot. - `MarkSetType` -- specifies the type of markset to use during compaction. + The markset is the data structure used by compaction/gc to track live objects. The default value is `"map"`, which will use an in-memory map; if you are limited in memory (or indeed see compaction run out of memory), you can also specify `"badger"` which will use an disk backed markset, using badger. This will use From d6ebead75639544380b0d0f7dbd7460b4b00ac73 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 23 Jul 2021 16:30:55 +0300 Subject: [PATCH 15/15] add comment about the necessity of FileIO --- blockstore/splitstore/markset_badger.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/blockstore/splitstore/markset_badger.go b/blockstore/splitstore/markset_badger.go index 6e0e33821..ef67db213 100644 --- a/blockstore/splitstore/markset_badger.go +++ b/blockstore/splitstore/markset_badger.go @@ -64,6 +64,11 @@ func (e *BadgerMarkSetEnv) Create(name string, sizeHint int64) (MarkSet, error) opts.SyncWrites = false opts.CompactL0OnClose = false opts.Compression = options.None + // Note: We use FileIO for loading modes to avoid memory thrashing and interference + // between the system blockstore and the markset. + // It was observed that using the default memory mapped option resulted in + // significant interference and unacceptably high block validation times once the markset + // exceeded 1GB in size. opts.TableLoadingMode = options.FileIO opts.ValueLogLoadingMode = options.FileIO opts.Logger = &badgerLogger{