LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

refactor!: abstractions for snapshot and pruning; snapshot intervals eventually pruned; unit tests (#11496)

Browse Source
This commit is contained in:
Roman
2022-04-21 15:30:36 -04:00
committed by GitHub
parent d4dd44469f
commit 42f8d45b68
57 changed files with 2643 additions and 638 deletions
Download Patch File Download Diff File Expand all files Collapse all files
store/rootmulti/dbadapter.go
+6 -2
View File
@@ -3,6 +3,8 @@ package rootmulti
import (
"github.com/cosmos/cosmos-sdk/store/dbadapter"
"github.com/cosmos/cosmos-sdk/store/types"
pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
)
var commithash = []byte("FAKE_HASH")
@@ -30,8 +32,10 @@ func (cdsa commitDBStoreAdapter) LastCommitID() types.CommitID {
}
}
func (cdsa commitDBStoreAdapter) SetPruning(_ types.PruningOptions) {}
func (cdsa commitDBStoreAdapter) SetPruning(_ pruningtypes.PruningOptions) {}
// GetPruning is a no-op as pruning options cannot be directly set on this store.
// They must be set on the root commit multi-store.
func (cdsa commitDBStoreAdapter) GetPruning() types.PruningOptions { return types.PruningOptions{} }
func (cdsa commitDBStoreAdapter) GetPruning() pruningtypes.PruningOptions {
return pruningtypes.NewPruningOptions(pruningtypes.PruningUndefined)
}
store/rootmulti/proof_test.go
+3 -2
View File
@@ -5,6 +5,7 @@ import (
"github.com/stretchr/testify/require"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/store/iavl"
@@ -57,7 +58,7 @@ func TestVerifyIAVLStoreQueryProof(t *testing.T) {
func TestVerifyMultiStoreQueryProof(t *testing.T) {
// Create main tree for testing.
db := dbm.NewMemDB()
store := NewStore(db)
store := NewStore(db, log.NewNopLogger())
iavlStoreKey := types.NewKVStoreKey("iavlStoreKey")
store.MountStoreWithDB(iavlStoreKey, types.StoreTypeIAVL, nil)
@@ -112,7 +113,7 @@ func TestVerifyMultiStoreQueryProof(t *testing.T) {
func TestVerifyMultiStoreQueryProofAbsence(t *testing.T) {
// Create main tree for testing.
db := dbm.NewMemDB()
store := NewStore(db)
store := NewStore(db, log.NewNopLogger())
iavlStoreKey := types.NewKVStoreKey("iavlStoreKey")
store.MountStoreWithDB(iavlStoreKey, types.StoreTypeIAVL, nil)
store/rootmulti/snapshot_test.go
+5 -4
View File
@@ -18,11 +18,12 @@ import (
"github.com/cosmos/cosmos-sdk/store/iavl"
"github.com/cosmos/cosmos-sdk/store/rootmulti"
"github.com/cosmos/cosmos-sdk/store/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
)
func newMultiStoreWithGeneratedData(db dbm.DB, stores uint8, storeKeys uint64) *rootmulti.Store {
multiStore := rootmulti.NewStore(db)
multiStore := rootmulti.NewStore(db, log.NewNopLogger())
r := rand.New(rand.NewSource(49872768940)) // Fixed seed for deterministic tests
keys := []*types.KVStoreKey{}
@@ -54,7 +55,7 @@ func newMultiStoreWithGeneratedData(db dbm.DB, stores uint8, storeKeys uint64) *
}
func newMultiStoreWithMixedMounts(db dbm.DB) *rootmulti.Store {
store := rootmulti.NewStore(db)
store := rootmulti.NewStore(db, log.NewNopLogger())
store.MountStoreWithDB(types.NewKVStoreKey("iavl1"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB(types.NewKVStoreKey("iavl2"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB(types.NewKVStoreKey("iavl3"), types.StoreTypeIAVL, nil)
@@ -234,7 +235,7 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
b.StartTimer()
for i := 0; i < b.N; i++ {
target := rootmulti.NewStore(dbm.NewMemDB())
target := rootmulti.NewStore(dbm.NewMemDB(), log.NewNopLogger())
for _, key := range source.StoreKeysByName() {
target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
}
@@ -269,7 +270,7 @@ func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys ui
b.StartTimer()
for i := 0; i < b.N; i++ {
target := rootmulti.NewStore(dbm.NewMemDB())
target := rootmulti.NewStore(dbm.NewMemDB(), log.NewNopLogger())
for _, key := range source.StoreKeysByName() {
target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
}
store/rootmulti/store.go
+110 -109
View File
@@ -1,7 +1,6 @@
package rootmulti
import (
"encoding/binary"
"fmt"
"io"
"math"
@@ -14,8 +13,11 @@ import (
gogotypes "github.com/gogo/protobuf/types"
"github.com/pkg/errors"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/pruning"
pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
"github.com/cosmos/cosmos-sdk/store/cachemulti"
"github.com/cosmos/cosmos-sdk/store/dbadapter"
@@ -30,7 +32,6 @@ import (
const (
latestVersionKey = "s/latest"
pruneHeightsKey = "s/pruneheights"
commitInfoKeyFmt = "s/%d" // s/<version>
)
@@ -39,14 +40,14 @@ const (
// the CommitMultiStore interface.
type Store struct {
db dbm.DB
logger log.Logger
lastCommitInfo *types.CommitInfo
pruningOpts types.PruningOptions
pruningManager *pruning.Manager
iavlCacheSize int
storesParams map[types.StoreKey]storeParams
stores map[types.StoreKey]types.CommitKVStore
keysByName map[string]types.StoreKey
lazyLoading bool
pruneHeights []int64
initialVersion int64
removalMap map[types.StoreKey]bool
@@ -68,30 +69,36 @@ var (
// store will be created with a PruneNothing pruning strategy by default. After
// a store is created, KVStores must be mounted and finally LoadLatestVersion or
// LoadVersion must be called.
func NewStore(db dbm.DB) *Store {
func NewStore(db dbm.DB, logger log.Logger) *Store {
return &Store{
db: db,
pruningOpts: types.PruneNothing,
iavlCacheSize: iavl.DefaultIAVLCacheSize,
storesParams: make(map[types.StoreKey]storeParams),
stores: make(map[types.StoreKey]types.CommitKVStore),
keysByName: make(map[string]types.StoreKey),
pruneHeights: make([]int64, 0),
listeners: make(map[types.StoreKey][]types.WriteListener),
removalMap: make(map[types.StoreKey]bool),
db: db,
logger: logger,
iavlCacheSize: iavl.DefaultIAVLCacheSize,
storesParams: make(map[types.StoreKey]storeParams),
stores: make(map[types.StoreKey]types.CommitKVStore),
keysByName: make(map[string]types.StoreKey),
listeners: make(map[types.StoreKey][]types.WriteListener),
removalMap: make(map[types.StoreKey]bool),
pruningManager: pruning.NewManager(db, logger),
}
}
// GetPruning fetches the pruning strategy from the root store.
func (rs *Store) GetPruning() types.PruningOptions {
return rs.pruningOpts
func (rs *Store) GetPruning() pruningtypes.PruningOptions {
return rs.pruningManager.GetOptions()
}
// SetPruning sets the pruning strategy on the root store and all the sub-stores.
// Note, calling SetPruning on the root store prior to LoadVersion or
// LoadLatestVersion performs a no-op as the stores aren't mounted yet.
func (rs *Store) SetPruning(pruningOpts types.PruningOptions) {
rs.pruningOpts = pruningOpts
func (rs *Store) SetPruning(pruningOpts pruningtypes.PruningOptions) {
rs.pruningManager.SetOptions(pruningOpts)
}
// SetSnapshotInterval sets the interval at which the snapshots are taken.
// It is used by the store to determine which heights to retain until after the snapshot is complete.
func (rs *Store) SetSnapshotInterval(snapshotInterval uint64) {
rs.pruningManager.SetSnapshotInterval(snapshotInterval)
}
func (rs *Store) SetIAVLCacheSize(cacheSize int) {
@@ -262,9 +269,8 @@ func (rs *Store) loadVersion(ver int64, upgrades *types.StoreUpgrades) error {
rs.stores = newStores
// load any pruned heights we missed from disk to be pruned on the next run
ph, err := getPruningHeights(rs.db)
if err == nil && len(ph) > 0 {
rs.pruneHeights = ph
if err := rs.pruningManager.LoadPruningHeights(rs.db); err != nil {
return err
}
return nil
@@ -309,6 +315,14 @@ func moveKVStoreData(oldDB types.KVStore, newDB types.KVStore) error {
return deleteKVStore(oldDB)
}
// PruneSnapshotHeight prunes the given height according to the prune strategy.
// If PruneNothing, this is a no-op.
// If other strategy, this height is persisted until it is
// less than <current height> - KeepRecent and <current height> % Interval == 0
func (rs *Store) PruneSnapshotHeight(height int64) {
rs.pruningManager.HandleHeightSnapshot(height)
}
// SetInterBlockCache sets the Store's internal inter-block (persistent) cache.
// When this is defined, all CommitKVStores will be wrapped with their respective
// inter-block cache.
@@ -403,6 +417,7 @@ func (rs *Store) Commit() types.CommitID {
}
rs.lastCommitInfo = commitStores(version, rs.stores, rs.removalMap)
defer rs.flushMetadata(rs.db, version, rs.lastCommitInfo)
// remove remnants of removed stores
for sk := range rs.removalMap {
@@ -412,54 +427,19 @@ func (rs *Store) Commit() types.CommitID {
delete(rs.keysByName, sk.Name())
}
}
// reset the removalMap
rs.removalMap = make(map[types.StoreKey]bool)
// Determine if pruneHeight height needs to be added to the list of heights to
// be pruned, where pruneHeight = (commitHeight - 1) - KeepRecent.
if rs.pruningOpts.Interval > 0 && int64(rs.pruningOpts.KeepRecent) < previousHeight {
pruneHeight := previousHeight - int64(rs.pruningOpts.KeepRecent)
rs.pruneHeights = append(rs.pruneHeights, pruneHeight)
if err := rs.handlePruning(version); err != nil {
panic(err)
}
// batch prune if the current height is a pruning interval height
if rs.pruningOpts.Interval > 0 && version%int64(rs.pruningOpts.Interval) == 0 {
rs.pruneStores()
}
flushMetadata(rs.db, version, rs.lastCommitInfo, rs.pruneHeights)
return types.CommitID{
Version: version,
Hash: rs.lastCommitInfo.Hash(),
}
}
// pruneStores will batch delete a list of heights from each mounted sub-store.
// Afterwards, pruneHeights is reset.
func (rs *Store) pruneStores() {
if len(rs.pruneHeights) == 0 {
return
}
for key, store := range rs.stores {
if store.GetStoreType() == types.StoreTypeIAVL {
// If the store is wrapped with an inter-block cache, we must first unwrap
// it to get the underlying IAVL store.
store = rs.GetCommitKVStore(key)
if err := store.(*iavl.Store).DeleteVersions(rs.pruneHeights...); err != nil {
if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist {
panic(err)
}
}
}
}
rs.pruneHeights = make([]int64, 0)
}
// CacheWrap implements CacheWrapper/Store/CommitStore.
func (rs *Store) CacheWrap() types.CacheWrap {
return rs.CacheMultiStore().(types.CacheWrap)
@@ -553,7 +533,51 @@ func (rs *Store) GetKVStore(key types.StoreKey) types.KVStore {
return store
}
// GetStoreByName performs a lookup of a StoreKey given a store name typically
func (rs *Store) handlePruning(version int64) error {
rs.pruningManager.HandleHeight(version - 1) // we should never prune the current version.
if !rs.pruningManager.ShouldPruneAtHeight(version) {
return nil
}
rs.logger.Info("prune start", "height", version)
defer rs.logger.Info("prune end", "height", version)
return rs.pruneStores()
}
func (rs *Store) pruneStores() error {
pruningHeights, err := rs.pruningManager.GetFlushAndResetPruningHeights()
if err != nil {
return err
}
if len(pruningHeights) == 0 {
rs.logger.Debug("pruning skipped; no heights to prune")
return nil
}
rs.logger.Debug("pruning heights", "heights", pruningHeights)
for key, store := range rs.stores {
// If the store is wrapped with an inter-block cache, we must first unwrap
// it to get the underlying IAVL store.
if store.GetStoreType() != types.StoreTypeIAVL {
continue
}
store = rs.GetCommitKVStore(key)
err := store.(*iavl.Store).DeleteVersions(pruningHeights...)
if err == nil {
continue
}
if errCause := errors.Cause(err); errCause != nil && errCause != iavltree.ErrVersionDoesNotExist {
return err
}
}
return nil
}
// getStoreByName performs a lookup of a StoreKey given a store name typically
// provided in a path. The StoreKey is then used to perform a lookup and return
// a Store. If the Store is wrapped in an inter-block cache, it will be unwrapped
// prior to being returned. If the StoreKey does not exist, nil is returned.
@@ -666,7 +690,7 @@ func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
if height == 0 {
return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0")
}
if height > uint64(rs.LastCommitID().Version) {
if height > uint64(getLatestVersion(rs.db)) {
return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height)
}
@@ -819,7 +843,7 @@ loop:
importer.Close()
}
flushMetadata(rs.db, int64(height), rs.buildCommitInfo(int64(height)), []int64{})
rs.flushMetadata(rs.db, int64(height), rs.buildCommitInfo(int64(height)))
return snapshotItem, rs.LoadLatestVersion()
}
@@ -910,9 +934,11 @@ func (rs *Store) RollbackToVersion(target int64) int64 {
return current
}
for ; current > target; current-- {
rs.pruneHeights = append(rs.pruneHeights, current)
rs.pruningManager.HandleHeight(current)
}
if err := rs.pruneStores(); err != nil {
panic(err)
}
rs.pruneStores()
// update latest height
bz, err := gogotypes.StdInt64Marshal(current)
@@ -924,6 +950,25 @@ func (rs *Store) RollbackToVersion(target int64) int64 {
return current
}
func (rs *Store) flushMetadata(db dbm.DB, version int64, cInfo *types.CommitInfo) {
rs.logger.Debug("flushing metadata", "height", version)
batch := db.NewBatch()
defer batch.Close()
if cInfo != nil {
flushCommitInfo(batch, version, cInfo)
} else {
rs.logger.Debug("commitInfo is nil, not flushed", "height", version)
}
flushLatestVersion(batch, version)
if err := batch.WriteSync(); err != nil {
panic(fmt.Errorf("error on batch write %w", err))
}
rs.logger.Debug("flushing metadata finished", "height", version)
}
type storeParams struct {
key types.StoreKey
db dbm.DB
@@ -996,7 +1041,7 @@ func getCommitInfo(db dbm.DB, ver int64) (*types.CommitInfo, error) {
return cInfo, nil
}
func setCommitInfo(batch dbm.Batch, version int64, cInfo *types.CommitInfo) {
func flushCommitInfo(batch dbm.Batch, version int64, cInfo *types.CommitInfo) {
bz, err := cInfo.Marshal()
if err != nil {
panic(err)
@@ -1006,7 +1051,7 @@ func setCommitInfo(batch dbm.Batch, version int64, cInfo *types.CommitInfo) {
batch.Set([]byte(cInfoKey), bz)
}
func setLatestVersion(batch dbm.Batch, version int64) {
func flushLatestVersion(batch dbm.Batch, version int64) {
bz, err := gogotypes.StdInt64Marshal(version)
if err != nil {
panic(err)
@@ -1014,47 +1059,3 @@ func setLatestVersion(batch dbm.Batch, version int64) {
batch.Set([]byte(latestVersionKey), bz)
}
func setPruningHeights(batch dbm.Batch, pruneHeights []int64) {
bz := make([]byte, 0)
for _, ph := range pruneHeights {
buf := make([]byte, 8)
binary.BigEndian.PutUint64(buf, uint64(ph))
bz = append(bz, buf...)
}
batch.Set([]byte(pruneHeightsKey), bz)
}
func getPruningHeights(db dbm.DB) ([]int64, error) {
bz, err := db.Get([]byte(pruneHeightsKey))
if err != nil {
return nil, fmt.Errorf("failed to get pruned heights: %w", err)
}
if len(bz) == 0 {
return nil, errors.New("no pruned heights found")
}
prunedHeights := make([]int64, len(bz)/8)
i, offset := 0, 0
for offset < len(bz) {
prunedHeights[i] = int64(binary.BigEndian.Uint64(bz[offset : offset+8]))
i++
offset += 8
}
return prunedHeights, nil
}
func flushMetadata(db dbm.DB, version int64, cInfo *types.CommitInfo, pruneHeights []int64) {
batch := db.NewBatch()
defer batch.Close()
setCommitInfo(batch, version, cInfo)
setLatestVersion(batch, version)
setPruningHeights(batch, pruneHeights)
if err := batch.Write(); err != nil {
panic(fmt.Errorf("error on batch write %w", err))
}
}
store/rootmulti/store_test.go
+110 -43
View File
@@ -8,10 +8,12 @@ import (
"github.com/stretchr/testify/require"
abci "github.com/tendermint/tendermint/abci/types"
"github.com/tendermint/tendermint/libs/log"
dbm "github.com/tendermint/tm-db"
"github.com/cosmos/cosmos-sdk/codec"
codecTypes "github.com/cosmos/cosmos-sdk/codec/types"
pruningtypes "github.com/cosmos/cosmos-sdk/pruning/types"
"github.com/cosmos/cosmos-sdk/store/cachemulti"
"github.com/cosmos/cosmos-sdk/store/iavl"
sdkmaps "github.com/cosmos/cosmos-sdk/store/internal/maps"
@@ -22,13 +24,13 @@ import (
func TestStoreType(t *testing.T) {
db := dbm.NewMemDB()
store := NewStore(db)
store := NewStore(db, log.NewNopLogger())
store.MountStoreWithDB(types.NewKVStoreKey("store1"), types.StoreTypeIAVL, db)
}
func TestGetCommitKVStore(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.PruneDefault)
ms := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningDefault))
err := ms.LoadLatestVersion()
require.Nil(t, err)
@@ -45,7 +47,7 @@ func TestGetCommitKVStore(t *testing.T) {
func TestStoreMount(t *testing.T) {
db := dbm.NewMemDB()
store := NewStore(db)
store := NewStore(db, log.NewNopLogger())
key1 := types.NewKVStoreKey("store1")
key2 := types.NewKVStoreKey("store2")
@@ -61,7 +63,7 @@ func TestStoreMount(t *testing.T) {
func TestCacheMultiStore(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.PruneNothing)
ms := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
cacheMulti := ms.CacheMultiStore()
require.IsType(t, cachemulti.Store{}, cacheMulti)
@@ -69,7 +71,7 @@ func TestCacheMultiStore(t *testing.T) {
func TestCacheMultiStoreWithVersion(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.PruneNothing)
ms := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := ms.LoadLatestVersion()
require.Nil(t, err)
@@ -106,7 +108,7 @@ func TestCacheMultiStoreWithVersion(t *testing.T) {
func TestHashStableWithEmptyCommit(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.PruneNothing)
ms := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := ms.LoadLatestVersion()
require.Nil(t, err)
@@ -130,7 +132,7 @@ func TestHashStableWithEmptyCommit(t *testing.T) {
func TestMultistoreCommitLoad(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
store := newMultiStoreWithMounts(db, types.PruneNothing)
store := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := store.LoadLatestVersion()
require.Nil(t, err)
@@ -155,7 +157,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
}
// Load the latest multistore again and check version.
store = newMultiStoreWithMounts(db, types.PruneNothing)
store = newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = store.LoadLatestVersion()
require.Nil(t, err)
commitID = getExpectedCommitID(store, nCommits)
@@ -168,7 +170,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
// Load an older multistore and check version.
ver := nCommits - 1
store = newMultiStoreWithMounts(db, types.PruneNothing)
store = newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = store.LoadVersion(ver)
require.Nil(t, err)
commitID = getExpectedCommitID(store, ver)
@@ -177,7 +179,7 @@ func TestMultistoreCommitLoad(t *testing.T) {
func TestMultistoreLoadWithUpgrade(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
store := newMultiStoreWithMounts(db, types.PruneNothing)
store := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := store.LoadLatestVersion()
require.Nil(t, err)
@@ -212,7 +214,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
checkContains(t, ci.StoreInfos, []string{"store1", "store2", "store3"})
// Load without changes and make sure it is sensible
store = newMultiStoreWithMounts(db, types.PruneNothing)
store = newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = store.LoadLatestVersion()
require.Nil(t, err)
@@ -225,7 +227,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
require.Equal(t, v2, s2.Get(k2))
// now, let's load with upgrades...
restore, upgrades := newMultiStoreWithModifiedMounts(db, types.PruneNothing)
restore, upgrades := newMultiStoreWithModifiedMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = restore.LoadLatestVersionAndUpgrade(upgrades)
require.Nil(t, err)
@@ -270,7 +272,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
migratedID := restore.Commit()
require.Equal(t, migratedID.Version, int64(2))
reload, _ := newMultiStoreWithModifiedMounts(db, types.PruneNothing)
reload, _ := newMultiStoreWithModifiedMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = reload.LoadLatestVersion()
require.Nil(t, err)
require.Equal(t, migratedID, reload.LastCommitID())
@@ -319,10 +321,7 @@ func TestParsePath(t *testing.T) {
func TestMultiStoreRestart(t *testing.T) {
db := dbm.NewMemDB()
pruning := types.PruningOptions{
KeepRecent: 2,
Interval: 1,
}
pruning := pruningtypes.NewCustomPruningOptions(2, 1)
multi := newMultiStoreWithMounts(db, pruning)
err := multi.LoadLatestVersion()
require.Nil(t, err)
@@ -401,7 +400,7 @@ func TestMultiStoreRestart(t *testing.T) {
func TestMultiStoreQuery(t *testing.T) {
db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := multi.LoadLatestVersion()
require.Nil(t, err)
@@ -428,7 +427,7 @@ func TestMultiStoreQuery(t *testing.T) {
ver := cid.Version
// Reload multistore from database
multi = newMultiStoreWithMounts(db, types.PruneNothing)
multi = newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err = multi.LoadLatestVersion()
require.Nil(t, err)
@@ -473,15 +472,15 @@ func TestMultiStore_Pruning(t *testing.T) {
testCases := []struct {
name string
numVersions int64
po types.PruningOptions
po pruningtypes.PruningOptions
deleted []int64
saved []int64
}{
{"prune nothing", 10, types.PruneNothing, nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
{"prune everything", 10, types.PruneEverything, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, []int64{10}},
{"prune some; no batch", 10, types.NewPruningOptions(2, 1), []int64{1, 2, 4, 5, 7}, []int64{3, 6, 8, 9, 10}},
{"prune some; small batch", 10, types.NewPruningOptions(2, 3), []int64{1, 2, 4, 5}, []int64{3, 6, 7, 8, 9, 10}},
{"prune some; large batch", 10, types.NewPruningOptions(2, 11), nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
{"prune nothing", 10, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing), nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
{"prune everything", 10, pruningtypes.NewPruningOptions(pruningtypes.PruningEverything), []int64{1, 2, 3, 4, 5, 6, 7, 8, 9}, []int64{10}},
{"prune some; no batch", 10, pruningtypes.NewCustomPruningOptions(2, 1), []int64{1, 2, 4, 5, 7}, []int64{3, 6, 8, 9, 10}},
{"prune some; small batch", 10, pruningtypes.NewCustomPruningOptions(2, 3), []int64{1, 2, 4, 5}, []int64{3, 6, 7, 8, 9, 10}},
{"prune some; large batch", 10, pruningtypes.NewCustomPruningOptions(2, 11), nil, []int64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}},
}
for _, tc := range testCases {
@@ -509,9 +508,64 @@ func TestMultiStore_Pruning(t *testing.T) {
}
}
func TestMultiStore_Pruning_SameHeightsTwice(t *testing.T) {
const (
numVersions int64 = 10
keepRecent uint64 = 2
interval uint64 = 10
)
expectedHeights := []int64{}
for i := int64(1); i < numVersions-int64(keepRecent); i++ {
expectedHeights = append(expectedHeights, i)
}
db := dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, pruningtypes.NewCustomPruningOptions(keepRecent, interval))
require.NoError(t, ms.LoadLatestVersion())
var lastCommitInfo types.CommitID
for i := int64(0); i < numVersions; i++ {
lastCommitInfo = ms.Commit()
}
require.Equal(t, numVersions, lastCommitInfo.Version)
for v := int64(1); v < numVersions-int64(keepRecent); v++ {
err := ms.LoadVersion(v)
require.Error(t, err, "expected error when loading pruned height: %d", v)
}
for v := int64(numVersions - int64(keepRecent)); v < numVersions; v++ {
err := ms.LoadVersion(v)
require.NoError(t, err, "expected no error when loading height: %d", v)
}
// Get latest
err := ms.LoadVersion(numVersions - 1)
require.NoError(t, err)
// Ensure already pruned heights were loaded
heights, err := ms.pruningManager.GetFlushAndResetPruningHeights()
require.NoError(t, err)
require.Equal(t, expectedHeights, heights)
require.NoError(t, ms.pruningManager.LoadPruningHeights(db))
// Test pruning the same heights again
lastCommitInfo = ms.Commit()
require.Equal(t, numVersions, lastCommitInfo.Version)
// Ensure that can commit one more height with no panic
lastCommitInfo = ms.Commit()
require.Equal(t, numVersions+1, lastCommitInfo.Version)
}
func TestMultiStore_PruningRestart(t *testing.T) {
db := dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.NewPruningOptions(2, 11))
ms := newMultiStoreWithMounts(db, pruningtypes.NewCustomPruningOptions(2, 11))
ms.SetSnapshotInterval(3)
require.NoError(t, ms.LoadLatestVersion())
// Commit enough to build up heights to prune, where on the next block we should
@@ -523,19 +577,30 @@ func TestMultiStore_PruningRestart(t *testing.T) {
pruneHeights := []int64{1, 2, 4, 5, 7}
// ensure we've persisted the current batch of heights to prune to the store's DB
ph, err := getPruningHeights(ms.db)
err := ms.pruningManager.LoadPruningHeights(ms.db)
require.NoError(t, err)
require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ph)
actualHeightsToPrune, err := ms.pruningManager.GetFlushAndResetPruningHeights()
require.NoError(t, err)
require.Equal(t, len(pruneHeights), len(actualHeightsToPrune))
require.Equal(t, pruneHeights, actualHeightsToPrune)
// "restart"
ms = newMultiStoreWithMounts(db, types.NewPruningOptions(2, 11))
ms = newMultiStoreWithMounts(db, pruningtypes.NewCustomPruningOptions(2, 11))
ms.SetSnapshotInterval(3)
err = ms.LoadLatestVersion()
require.NoError(t, err)
require.Equal(t, []int64{1, 2, 3, 4, 5, 6, 7}, ms.pruneHeights)
actualHeightsToPrune, err = ms.pruningManager.GetFlushAndResetPruningHeights()
require.NoError(t, err)
require.Equal(t, pruneHeights, actualHeightsToPrune)
// commit one more block and ensure the heights have been pruned
ms.Commit()
require.Empty(t, ms.pruneHeights)
actualHeightsToPrune, err = ms.pruningManager.GetFlushAndResetPruningHeights()
require.NoError(t, err)
require.Empty(t, actualHeightsToPrune)
for _, v := range pruneHeights {
_, err := ms.CacheMultiStoreWithVersion(v)
@@ -545,7 +610,7 @@ func TestMultiStore_PruningRestart(t *testing.T) {
func TestSetInitialVersion(t *testing.T) {
db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
require.NoError(t, multi.LoadLatestVersion())
@@ -563,7 +628,7 @@ func TestSetInitialVersion(t *testing.T) {
func TestAddListenersAndListeningEnabled(t *testing.T) {
db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
testKey := types.NewKVStoreKey("listening_test_key")
enabled := multi.ListeningEnabled(testKey)
require.False(t, enabled)
@@ -594,7 +659,7 @@ var (
func TestGetListenWrappedKVStore(t *testing.T) {
buf := new(bytes.Buffer)
var db dbm.DB = dbm.NewMemDB()
ms := newMultiStoreWithMounts(db, types.PruneNothing)
ms := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
ms.LoadLatestVersion()
mockListeners := []types.WriteListener{types.NewStoreKVPairWriteListener(buf, testMarshaller)}
ms.AddListeners(testStoreKey1, mockListeners)
@@ -637,6 +702,7 @@ func TestGetListenWrappedKVStore(t *testing.T) {
StoreKey: testStoreKey2.Name(),
Delete: false,
})
require.NoError(t, err)
kvPairSet2Bytes := buf.Bytes()
buf.Reset()
require.Equal(t, expectedOutputKVPairSet2, kvPairSet2Bytes)
@@ -648,6 +714,7 @@ func TestGetListenWrappedKVStore(t *testing.T) {
StoreKey: testStoreKey2.Name(),
Delete: true,
})
require.NoError(t, err)
kvPairDelete2Bytes := buf.Bytes()
buf.Reset()
require.Equal(t, expectedOutputKVPairDelete2, kvPairDelete2Bytes)
@@ -668,7 +735,7 @@ func TestGetListenWrappedKVStore(t *testing.T) {
func TestCacheWraps(t *testing.T) {
db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
cacheWrapper := multi.CacheWrap()
require.IsType(t, cachemulti.Store{}, cacheWrapper)
@@ -682,7 +749,7 @@ func TestCacheWraps(t *testing.T) {
func TestTraceConcurrency(t *testing.T) {
db := dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := multi.LoadLatestVersion()
require.NoError(t, err)
@@ -732,7 +799,7 @@ func TestTraceConcurrency(t *testing.T) {
func TestCommitOrdered(t *testing.T) {
var db dbm.DB = dbm.NewMemDB()
multi := newMultiStoreWithMounts(db, types.PruneNothing)
multi := newMultiStoreWithMounts(db, pruningtypes.NewPruningOptions(pruningtypes.PruningNothing))
err := multi.LoadLatestVersion()
require.Nil(t, err)
@@ -773,9 +840,9 @@ var (
testStoreKey3 = types.NewKVStoreKey("store3")
)
func newMultiStoreWithMounts(db dbm.DB, pruningOpts types.PruningOptions) *Store {
store := NewStore(db)
store.pruningOpts = pruningOpts
func newMultiStoreWithMounts(db dbm.DB, pruningOpts pruningtypes.PruningOptions) *Store {
store := NewStore(db, log.NewNopLogger())
store.SetPruning(pruningOpts)
store.MountStoreWithDB(testStoreKey1, types.StoreTypeIAVL, nil)
store.MountStoreWithDB(testStoreKey2, types.StoreTypeIAVL, nil)
@@ -784,9 +851,9 @@ func newMultiStoreWithMounts(db dbm.DB, pruningOpts types.PruningOptions) *Store
return store
}
func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions) (*Store, *types.StoreUpgrades) {
store := NewStore(db)
store.pruningOpts = pruningOpts
func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts pruningtypes.PruningOptions) (*Store, *types.StoreUpgrades) {
store := NewStore(db, log.NewNopLogger())
store.SetPruning(pruningOpts)
store.MountStoreWithDB(types.NewKVStoreKey("store1"), types.StoreTypeIAVL, nil)
store.MountStoreWithDB(types.NewKVStoreKey("restore2"), types.StoreTypeIAVL, nil)