From ea672d40f51dda78677c8826270d713a4dc43bbc Mon Sep 17 00:00:00 2001 From: Sai Kumar <17549398+gsk967@users.noreply.github.com> Date: Sun, 20 Mar 2022 08:47:23 +0530 Subject: [PATCH] feat: state migration from IAVL to SMT (ADR-040) (#10962) State migration from iavl to smt This migration works on upgrade handler ```go app.registerUpgradeHandlers() func (app *SimApp) registerUpgradeHandlers() { // This is the upgrade plan name we used in the gov proposal. upgradeName := "iavl-to-smt-upgrade" app.UpgradeKeeper.SetUpgradeHandler(upgradeName, func(ctx sdk.Context, plan upgradetypes.Plan, _ module.VersionMap) (module.VersionMap, error) { fromVM := map[string]uint64{ ... } err := store2.MigrateV2(iavlStore, v2Store) if err != nil { return fromVM, err } return app.mm.RunMigrations(ctx, app.configurator, fromVM) }) } ``` ## Description Closes: #XXXX --- ### Author Checklist *All items are required. Please add a note to the item if the item is not applicable and please add links to any relevant follow up issues.* I have... - [ ] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] added `!` to the type prefix if API or client breaking change - [ ] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting)) - [ ] provided a link to the relevant issue or specification - [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules) - [ ] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing) - [ ] added a changelog entry to `CHANGELOG.md` - [ ] included comments for [documenting Go code](https://blog.golang.org/godoc) - [ ] updated the relevant documentation or specification - [ ] reviewed "Files changed" and left comments if necessary - [ ] confirmed all CI checks have passed ### Reviewers Checklist *All items are required. Please add a note if the item is not applicable and please add your handle next to the items reviewed if you only reviewed selected items.* I have... - [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] confirmed `!` in the type prefix if API or client breaking change - [ ] confirmed all author checklist items have been addressed - [ ] reviewed state machine logic - [ ] reviewed API design and naming - [ ] reviewed documentation is accurate - [ ] reviewed tests and test coverage - [ ] manually tested (if applicable) --- CHANGELOG.md | 1 + store/rootmulti/snapshot_test.go | 7 +- store/rootmulti/store.go | 6 +- store/v2alpha1/multi/migration.go | 68 ++++++++++++++++ store/v2alpha1/multi/migration_test.go | 106 +++++++++++++++++++++++++ 5 files changed, 182 insertions(+), 6 deletions(-) create mode 100644 store/v2alpha1/multi/migration.go create mode 100644 store/v2alpha1/multi/migration_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 136af43452..56d1326209 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#11179](https://github.com/cosmos/cosmos-sdk/pull/11179) Add state rollback command. * [\#10794](https://github.com/cosmos/cosmos-sdk/pull/10794) ADR-040: Add State Sync to V2 Store * [\#11234](https://github.com/cosmos/cosmos-sdk/pull/11234) Add `GRPCClient` field to Client Context. If `GRPCClient` field is set to nil, the `Invoke` method would use ABCI query, otherwise use gprc. +* [\#10962](https://github.com/cosmos/cosmos-sdk/pull/10962) ADR-040: Add state migration from iavl (v1Store) to smt (v2Store) * (types) [\#10948](https://github.com/cosmos/cosmos-sdk/issues/10948) Add `app-db-backend` to the `app.toml` config to replace the compile-time `types.DBbackend` variable. ### API Breaking Changes diff --git a/store/rootmulti/snapshot_test.go b/store/rootmulti/snapshot_test.go index cbdb481a05..92e1311fdd 100644 --- a/store/rootmulti/snapshot_test.go +++ b/store/rootmulti/snapshot_test.go @@ -210,7 +210,8 @@ func TestMultistoreSnapshotRestore(t *testing.T) { require.Equal(t, *dummyExtensionItem.GetExtension(), *nextItem.GetExtension()) assert.Equal(t, source.LastCommitID(), target.LastCommitID()) - for key, sourceStore := range source.GetStores() { + for _, key := range source.StoreKeysByName() { + sourceStore := source.GetStoreByName(key.Name()).(types.CommitKVStore) targetStore := target.GetStoreByName(key.Name()).(types.CommitKVStore) switch sourceStore.GetStoreType() { case types.StoreTypeTransient: @@ -234,7 +235,7 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for key := range source.GetStores() { + for _, key := range source.StoreKeysByName() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() @@ -269,7 +270,7 @@ func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys ui for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for key := range source.GetStores() { + for _, key := range source.StoreKeysByName() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index b264352ff2..bd9502187c 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -148,9 +148,9 @@ func (rs *Store) GetCommitKVStore(key types.StoreKey) types.CommitKVStore { return rs.stores[key] } -// GetStores returns mounted stores -func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore { - return rs.stores +// StoreKeysByName returns mapping storeNames -> StoreKeys +func (rs *Store) StoreKeysByName() map[string]types.StoreKey { + return rs.keysByName } // LoadLatestVersionAndUpgrade implements CommitMultiStore diff --git a/store/v2alpha1/multi/migration.go b/store/v2alpha1/multi/migration.go new file mode 100644 index 0000000000..7481b421ac --- /dev/null +++ b/store/v2alpha1/multi/migration.go @@ -0,0 +1,68 @@ +package multi + +import ( + dbm "github.com/cosmos/cosmos-sdk/db" + "github.com/cosmos/cosmos-sdk/store/iavl" + "github.com/cosmos/cosmos-sdk/store/mem" + v1Store "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/cosmos/cosmos-sdk/store/transient" + "github.com/cosmos/cosmos-sdk/store/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" +) + +// MigrateFromV1 will migrate the state from iavl to smt +func MigrateFromV1(rootMultiStore *v1Store.Store, store2db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { + type namedStore struct { + *iavl.Store + name string + } + var stores []namedStore + for _, storeKey := range rootMultiStore.StoreKeysByName() { + keyName := storeKey.Name() + switch store := rootMultiStore.GetStoreByName(keyName).(type) { + case *iavl.Store: + err := storeConfig.RegisterSubstore(keyName, types.StoreTypePersistent) + if err != nil { + return nil, err + } + stores = append(stores, namedStore{name: keyName, Store: store}) + case *transient.Store, *mem.Store: + continue + default: + return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to migrate store %q of type %T", keyName, store) + } + } + + // creating the new store of smt tree + rootStore, err := NewStore(store2db, storeConfig) + if err != nil { + return nil, err + } + + // if version is 0 there is no state data to commit + if rootMultiStore.LastCommitID().Version == 0 { + return rootStore, nil + } + + // iterate through the rootmulti stores and save the key/values into smt tree + for _, store := range stores { + subStore, err := rootStore.getSubstore(store.name) + if err != nil { + return nil, err + } + // iterate all iavl tree node key/values + iterator := store.Iterator(nil, nil) + for ; iterator.Valid(); iterator.Next() { + // set the iavl key,values into smt node + subStore.Set(iterator.Key(), iterator.Value()) + } + } + + // commit the all key/values from iavl to smt tree (SMT Store) + _, err = rootStore.commit(uint64(rootMultiStore.LastCommitID().Version)) + if err != nil { + return nil, err + } + + return rootStore, nil +} diff --git a/store/v2alpha1/multi/migration_test.go b/store/v2alpha1/multi/migration_test.go new file mode 100644 index 0000000000..09f1f74b5a --- /dev/null +++ b/store/v2alpha1/multi/migration_test.go @@ -0,0 +1,106 @@ +package multi + +import ( + "encoding/binary" + "fmt" + "math/rand" + "testing" + + "github.com/cosmos/cosmos-sdk/db/memdb" + "github.com/cosmos/cosmos-sdk/store/iavl" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" +) + +func TestMigrationV2(t *testing.T) { + r := rand.New(rand.NewSource(49872768940)) + + // setup a rootmulti store + db := dbm.NewMemDB() + v1Store := rootmulti.NewStore(db) + + // mount the kvStores + var keys []*types.KVStoreKey + for i := uint8(0); i < 10; i++ { + key := types.NewKVStoreKey(fmt.Sprintf("store%v", i)) + v1Store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + keys = append(keys, key) + } + + err := v1Store.LoadLatestVersion() + require.Nil(t, err) + + // setup a random test data + for _, key := range keys { + store := v1Store.GetStore(key).(*iavl.Store) + store.Set([]byte("temp_data"), []byte("one")) + + for i := 0; i < len(keys); i++ { + k := make([]byte, 8) + v := make([]byte, 1024) + binary.BigEndian.PutUint64(k, uint64(i)) + _, err := r.Read(v) + if err != nil { + panic(err) + } + store.Set(k, v) + } + } + + testCases := []struct { + testName string + emptyStore bool + }{ + { + "Migration With Empty Store", + true, + }, + { + "Migration From Root Multi Store (IAVL) to SMT ", + false, + }, + } + + for _, testCase := range testCases { + if !testCase.emptyStore { + v1Store.Commit() + } + + // setup a new root store of smt + db2 := memdb.NewDB() + storeConfig := DefaultStoreConfig() + // migrating the iavl store (v1) to smt store (v2) + v2Store, err := MigrateFromV1(v1Store, db2, storeConfig) + require.NoError(t, err) + + for _, key := range keys { + v2StoreKVStore := v2Store.GetKVStore(key) + if testCase.emptyStore { + // check the empty store + require.Nil(t, v2StoreKVStore.Get([]byte("temp_data"))) + } else { + require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) + } + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + } + err = v2Store.Close() + require.NoError(t, err) + } +} + +// TestMigrateV2ForEmptyStore checking empty store migration +func TestMigrateV2ForEmptyStore(t *testing.T) { + // setup a rootmulti store + db := dbm.NewMemDB() + v1Store := rootmulti.NewStore(db) + err := v1Store.LoadLatestVersion() + require.Nil(t, err) + db2 := memdb.NewDB() + storeConfig := DefaultStoreConfig() + // migrating the iavl store (v1) to smt store (v2) + v2Store, err := MigrateFromV1(v1Store, db2, storeConfig) + require.NoError(t, err) + require.Equal(t, v2Store.LastCommitID(), v1Store.LastCommitID()) +}