fix lotus-shed datastore commands.

This commit is contained in:
Raúl Kripalani
2020-11-01 13:50:41 +00:00
parent a16d7f221e
commit d8d85373f5
3 changed files with 79 additions and 46 deletions
+52
View File
@@ -0,0 +1,52 @@
package repo
import badgerbs "github.com/filecoin-project/lotus/lib/blockstore/badger"
// BadgerBlockstoreOptions returns the badger options to apply for the provided
// domain.
func BadgerBlockstoreOptions(domain BlockstoreDomain, path string, readonly bool) (badgerbs.Options, error) {
if domain != BlockstoreChain {
return badgerbs.Options{}, ErrInvalidBlockstoreDomain
}
opts := badgerbs.DefaultOptions(path)
// Due to legacy usage of blockstore.Blockstore, over a datastore, all
// blocks are prefixed with this namespace. In the future, this can go away,
// in order to shorten keys, but it'll require a migration.
opts.Prefix = "/blocks/"
// Blockstore values are immutable; therefore we do not expect any
// conflicts to emerge.
opts.DetectConflicts = false
// This is to optimize the database on close so it can be opened
// read-only and efficiently queried. We don't do that and hanging on
// stop isn't nice.
opts.CompactL0OnClose = false
// The alternative is "crash on start and tell the user to fix it". This
// will truncate corrupt and unsynced data, which we don't guarantee to
// persist anyways.
opts.Truncate = true
// We mmap the index into memory, and access values from disk.
// Ideally the table loading mode would be settable by LSM level.
opts.ValueLogLoadingMode = badgerbs.FileIO
opts.TableLoadingMode = badgerbs.MemoryMap
// Embed only values < 128 bytes in the LSM; larger values in value logs.
opts.ValueThreshold = 128
// Reduce this from 64MiB to 16MiB. That means badger will hold on to
// 20MiB by default instead of 80MiB. This does not appear to have a
// significant performance hit.
opts.MaxTableSize = 16 << 20
// NOTE: The chain blockstore doesn't require any GC (blocks are never
// deleted). This will change if we move to a tiered blockstore.
opts.ReadOnly = readonly
return opts, nil
}
+6 -36
View File
@@ -286,49 +286,19 @@ func (fsr *fsLockedRepo) Close() error {
return err
}
// Blockstore returns a blockstore for the provided data domain.
func (fsr *fsLockedRepo) Blockstore(domain BlockstoreDomain) (blockstore.Blockstore, error) {
if domain != BlockstoreChain {
return nil, ErrInvalidBlockstoreDomain
}
path := fsr.join(filepath.Join(fsDatastore, "chain"))
opts := badgerbs.DefaultOptions(path)
// Due to legacy usage of blockstore.Blockstore, over a datastore, all
// blocks are prefixed with this namespace. In the future, this can go away,
// in order to shorten keys, but it'll require a migration.
opts.Prefix = "/blocks/"
// Blockstore values are immutable; therefore we do not expect any
// conflicts to emerge.
opts.DetectConflicts = false
// This is to optimize the database on close so it can be opened
// read-only and efficiently queried. We don't do that and hanging on
// stop isn't nice.
opts.CompactL0OnClose = false
// The alternative is "crash on start and tell the user to fix it". This
// will truncate corrupt and unsynced data, which we don't guarantee to
// persist anyways.
opts.Truncate = true
// We mmap the index into memory, and access values from disk.
// Ideally the table loading mode would be settable by LSM level.
opts.ValueLogLoadingMode = badgerbs.FileIO
opts.TableLoadingMode = badgerbs.MemoryMap
// Embed only values < 128 bytes in the LSM; larger values in value logs.
opts.ValueThreshold = 128
// Reduce this from 64MiB to 16MiB. That means badger will hold on to
// 20MiB by default instead of 80MiB. This does not appear to have a
// significant performance hit.
opts.MaxTableSize = 16 << 20
// NOTE: The chain blockstore doesn't require any GC (blocks are never
// deleted). This will change if we move to a tiered blockstore.
readonly := fsr.readonly
opts, err := BadgerBlockstoreOptions(domain, path, readonly)
if err != nil {
return nil, err
}
return badgerbs.Open(opts)
}