feat!: Add hooks to allow app modules to add things to state-sync (#10961)

## Description

Closes: #7340

- Support registering multiple snapshotters in snapshot manager.
- Append the extension snapshotters to existing snapshot stream.

~TODO: testing.~
- existing tests are fixed






---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [ ] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [ ] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [ ] provided a link to the relevant issue or specification
- [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [ ] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [ ] added a changelog entry to `CHANGELOG.md`
- [ ] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [ ] updated the relevant documentation or specification
- [ ] reviewed "Files changed" and left comments if necessary
- [ ] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

store/rootmulti/snapshot_test.go

@@ -0,0 +1,296 @@
+package rootmulti_test
+
+import (
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/cosmos/cosmos-sdk/snapshots"
+	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
+	"github.com/cosmos/cosmos-sdk/store/iavl"
+	"github.com/cosmos/cosmos-sdk/store/rootmulti"
+	"github.com/cosmos/cosmos-sdk/store/types"
+	dbm "github.com/tendermint/tm-db"
+)
+
+func newMultiStoreWithGeneratedData(db dbm.DB, stores uint8, storeKeys uint64) *rootmulti.Store {
+	multiStore := rootmulti.NewStore(db)
+	r := rand.New(rand.NewSource(49872768940)) // Fixed seed for deterministic tests
+
+	keys := []*types.KVStoreKey{}
+	for i := uint8(0); i < stores; i++ {
+		key := types.NewKVStoreKey(fmt.Sprintf("store%v", i))
+		multiStore.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
+		keys = append(keys, key)
+	}
+	multiStore.LoadLatestVersion()
+
+	for _, key := range keys {
+		store := multiStore.GetCommitKVStore(key).(*iavl.Store)
+		for i := uint64(0); i < storeKeys; i++ {
+			k := make([]byte, 8)
+			v := make([]byte, 1024)
+			binary.BigEndian.PutUint64(k, i)
+			_, err := r.Read(v)
+			if err != nil {
+				panic(err)
+			}
+			store.Set(k, v)
+		}
+	}
+
+	multiStore.Commit()
+	multiStore.LoadLatestVersion()
+
+	return multiStore
+}
+
+func newMultiStoreWithMixedMounts(db dbm.DB) *rootmulti.Store {
+	store := rootmulti.NewStore(db)
+	store.MountStoreWithDB(types.NewKVStoreKey("iavl1"), types.StoreTypeIAVL, nil)
+	store.MountStoreWithDB(types.NewKVStoreKey("iavl2"), types.StoreTypeIAVL, nil)
+	store.MountStoreWithDB(types.NewKVStoreKey("iavl3"), types.StoreTypeIAVL, nil)
+	store.MountStoreWithDB(types.NewTransientStoreKey("trans1"), types.StoreTypeTransient, nil)
+	store.LoadLatestVersion()
+
+	return store
+}
+
+func newMultiStoreWithMixedMountsAndBasicData(db dbm.DB) *rootmulti.Store {
+	store := newMultiStoreWithMixedMounts(db)
+	store1 := store.GetStoreByName("iavl1").(types.CommitKVStore)
+	store2 := store.GetStoreByName("iavl2").(types.CommitKVStore)
+	trans1 := store.GetStoreByName("trans1").(types.KVStore)
+
+	store1.Set([]byte("a"), []byte{1})
+	store1.Set([]byte("b"), []byte{1})
+	store2.Set([]byte("X"), []byte{255})
+	store2.Set([]byte("A"), []byte{101})
+	trans1.Set([]byte("x1"), []byte{91})
+	store.Commit()
+
+	store1.Set([]byte("b"), []byte{2})
+	store1.Set([]byte("c"), []byte{3})
+	store2.Set([]byte("B"), []byte{102})
+	store.Commit()
+
+	store2.Set([]byte("C"), []byte{103})
+	store2.Delete([]byte("X"))
+	trans1.Set([]byte("x2"), []byte{92})
+	store.Commit()
+
+	return store
+}
+
+func assertStoresEqual(t *testing.T, expect, actual types.CommitKVStore, msgAndArgs ...interface{}) {
+	assert.Equal(t, expect.LastCommitID(), actual.LastCommitID())
+	expectIter := expect.Iterator(nil, nil)
+	expectMap := map[string][]byte{}
+	for ; expectIter.Valid(); expectIter.Next() {
+		expectMap[string(expectIter.Key())] = expectIter.Value()
+	}
+	require.NoError(t, expectIter.Error())
+
+	actualIter := expect.Iterator(nil, nil)
+	actualMap := map[string][]byte{}
+	for ; actualIter.Valid(); actualIter.Next() {
+		actualMap[string(actualIter.Key())] = actualIter.Value()
+	}
+	require.NoError(t, actualIter.Error())
+
+	assert.Equal(t, expectMap, actualMap, msgAndArgs...)
+}
+
+func TestMultistoreSnapshot_Checksum(t *testing.T) {
+	// Chunks from different nodes must fit together, so all nodes must produce identical chunks.
+	// This checksum test makes sure that the byte stream remains identical. If the test fails
+	// without having changed the data (e.g. because the Protobuf or zlib encoding changes),
+	// snapshottypes.CurrentFormat must be bumped.
+	store := newMultiStoreWithGeneratedData(dbm.NewMemDB(), 5, 10000)
+	version := uint64(store.LastCommitID().Version)
+
+	testcases := []struct {
+		format      uint32
+		chunkHashes []string
+	}{
+		{1, []string{
+			"503e5b51b657055b77e88169fadae543619368744ad15f1de0736c0a20482f24",
+			"e1a0daaa738eeb43e778aefd2805e3dd720798288a410b06da4b8459c4d8f72e",
+			"aa048b4ee0f484965d7b3b06822cf0772cdcaad02f3b1b9055e69f2cb365ef3c",
+			"7921eaa3ed4921341e504d9308a9877986a879fe216a099c86e8db66fcba4c63",
+			"a4a864e6c02c9fca5837ec80dc84f650b25276ed7e4820cf7516ced9f9901b86",
+			"ca2879ac6e7205d257440131ba7e72bef784cd61642e32b847729e543c1928b9",
+		}},
+	}
+	for _, tc := range testcases {
+		tc := tc
+		t.Run(fmt.Sprintf("Format %v", tc.format), func(t *testing.T) {
+			ch := make(chan io.ReadCloser)
+			go func() {
+				streamWriter := snapshots.NewStreamWriter(ch)
+				defer streamWriter.Close()
+				require.NotNil(t, streamWriter)
+				err := store.Snapshot(version, streamWriter)
+				require.NoError(t, err)
+			}()
+			hashes := []string{}
+			hasher := sha256.New()
+			for chunk := range ch {
+				hasher.Reset()
+				_, err := io.Copy(hasher, chunk)
+				require.NoError(t, err)
+				hashes = append(hashes, hex.EncodeToString(hasher.Sum(nil)))
+			}
+			assert.Equal(t, tc.chunkHashes, hashes,
+				"Snapshot output for format %v has changed", tc.format)
+		})
+	}
+}
+
+func TestMultistoreSnapshot_Errors(t *testing.T) {
+	store := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB())
+
+	testcases := map[string]struct {
+		height     uint64
+		expectType error
+	}{
+		"0 height":       {0, nil},
+		"unknown height": {9, nil},
+	}
+	for name, tc := range testcases {
+		tc := tc
+		t.Run(name, func(t *testing.T) {
+			err := store.Snapshot(tc.height, nil)
+			require.Error(t, err)
+			if tc.expectType != nil {
+				assert.True(t, errors.Is(err, tc.expectType))
+			}
+		})
+	}
+}
+
+func TestMultistoreSnapshotRestore(t *testing.T) {
+	source := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB())
+	target := newMultiStoreWithMixedMounts(dbm.NewMemDB())
+	version := uint64(source.LastCommitID().Version)
+	require.EqualValues(t, 3, version)
+
+	chunks := make(chan io.ReadCloser, 100)
+	go func() {
+		streamWriter := snapshots.NewStreamWriter(chunks)
+		require.NotNil(t, streamWriter)
+		defer streamWriter.Close()
+		err := source.Snapshot(version, streamWriter)
+		require.NoError(t, err)
+	}()
+
+	streamReader, err := snapshots.NewStreamReader(chunks)
+	require.NoError(t, err)
+	_, err = target.Restore(version, snapshottypes.CurrentFormat, streamReader)
+	require.NoError(t, err)
+
+	assert.Equal(t, source.LastCommitID(), target.LastCommitID())
+	for key, sourceStore := range source.GetStores() {
+		targetStore := target.GetStoreByName(key.Name()).(types.CommitKVStore)
+		switch sourceStore.GetStoreType() {
+		case types.StoreTypeTransient:
+			assert.False(t, targetStore.Iterator(nil, nil).Valid(),
+				"transient store %v not empty", key.Name())
+		default:
+			assertStoresEqual(t, sourceStore, targetStore, "store %q not equal", key.Name())
+		}
+	}
+}
+
+func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
+	b.ReportAllocs()
+	b.StopTimer()
+	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
+	version := source.LastCommitID().Version
+	require.EqualValues(b, 1, version)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		target := rootmulti.NewStore(dbm.NewMemDB())
+		for key := range source.GetStores() {
+			target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
+		}
+		err := target.LoadLatestVersion()
+		require.NoError(b, err)
+		require.EqualValues(b, 0, target.LastCommitID().Version)
+
+		chunks := make(chan io.ReadCloser)
+		go func() {
+			streamWriter := snapshots.NewStreamWriter(chunks)
+			require.NotNil(b, streamWriter)
+			err := source.Snapshot(uint64(version), streamWriter)
+			require.NoError(b, err)
+		}()
+		for reader := range chunks {
+			_, err := io.Copy(io.Discard, reader)
+			require.NoError(b, err)
+			err = reader.Close()
+			require.NoError(b, err)
+		}
+	}
+}
+
+func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) {
+	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
+
+	b.ReportAllocs()
+	b.StopTimer()
+	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
+	version := uint64(source.LastCommitID().Version)
+	require.EqualValues(b, 1, version)
+	b.StartTimer()
+
+	for i := 0; i < b.N; i++ {
+		target := rootmulti.NewStore(dbm.NewMemDB())
+		for key := range source.GetStores() {
+			target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
+		}
+		err := target.LoadLatestVersion()
+		require.NoError(b, err)
+		require.EqualValues(b, 0, target.LastCommitID().Version)
+
+		chunks := make(chan io.ReadCloser)
+		go func() {
+			writer := snapshots.NewStreamWriter(chunks)
+			require.NotNil(b, writer)
+			err := source.Snapshot(version, writer)
+			require.NoError(b, err)
+		}()
+		reader, err := snapshots.NewStreamReader(chunks)
+		require.NoError(b, err)
+		_, err = target.Restore(version, snapshottypes.CurrentFormat, reader)
+		require.NoError(b, err)
+		require.Equal(b, source.LastCommitID(), target.LastCommitID())
+	}
+}
+
+func BenchmarkMultistoreSnapshot100K(b *testing.B) {
+	benchmarkMultistoreSnapshot(b, 10, 10000)
+}
+
+func BenchmarkMultistoreSnapshot1M(b *testing.B) {
+	benchmarkMultistoreSnapshot(b, 10, 100000)
+}
+
+func BenchmarkMultistoreSnapshotRestore100K(b *testing.B) {
+	benchmarkMultistoreSnapshotRestore(b, 10, 10000)
+}
+
+func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) {
+	benchmarkMultistoreSnapshotRestore(b, 10, 100000)
+}

store/rootmulti/store.go

@@ -1,8 +1,6 @@
 package rootmulti
 
 import (
-	"bufio"
-	"compress/zlib"
 	"encoding/binary"
 	"fmt"
 	"io"
@@ -18,7 +16,6 @@ import (
 	abci "github.com/tendermint/tendermint/abci/types"
 	dbm "github.com/tendermint/tm-db"
 
-	"github.com/cosmos/cosmos-sdk/snapshots"
 	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
 	"github.com/cosmos/cosmos-sdk/store/cachemulti"
 	"github.com/cosmos/cosmos-sdk/store/dbadapter"
@@ -35,11 +32,6 @@ const (
 	latestVersionKey = "s/latest"
 	pruneHeightsKey  = "s/pruneheights"
 	commitInfoKeyFmt = "s/%d" // s/<version>
-
-	// Do not change chunk size without new snapshot format (must be uniform across nodes)
-	snapshotChunkSize   = uint64(10e6)
-	snapshotBufferSize  = int(snapshotChunkSize)
-	snapshotMaxItemSize = int(64e6) // SDK has no key/value size limit, so we set an arbitrary limit
 )
 
 // Store is composed of many CommitStores. Name contrasts with
@@ -156,6 +148,11 @@ func (rs *Store) GetCommitKVStore(key types.StoreKey) types.CommitKVStore {
 	return rs.stores[key]
 }
 
+// GetStores returns mounted stores
+func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore {
+	return rs.stores
+}
+
 // LoadLatestVersionAndUpgrade implements CommitMultiStore
 func (rs *Store) LoadLatestVersionAndUpgrade(upgrades *types.StoreUpgrades) error {
 	ver := getLatestVersion(rs.db)
@@ -562,11 +559,11 @@ func (rs *Store) GetKVStore(key types.StoreKey) types.KVStore {
 	return store
 }
 
-// getStoreByName performs a lookup of a StoreKey given a store name typically
+// GetStoreByName performs a lookup of a StoreKey given a store name typically
 // provided in a path. The StoreKey is then used to perform a lookup and return
 // a Store. If the Store is wrapped in an inter-block cache, it will be unwrapped
 // prior to being returned. If the StoreKey does not exist, nil is returned.
-func (rs *Store) getStoreByName(name string) types.Store {
+func (rs *Store) GetStoreByName(name string) types.Store {
 	key := rs.keysByName[name]
 	if key == nil {
 		return nil
@@ -586,7 +583,7 @@ func (rs *Store) Query(req abci.RequestQuery) abci.ResponseQuery {
 		return sdkerrors.QueryResult(err, false)
 	}
 
-	store := rs.getStoreByName(storeName)
+	store := rs.GetStoreByName(storeName)
 	if store == nil {
 		return sdkerrors.QueryResult(sdkerrors.Wrapf(sdkerrors.ErrUnknownRequest, "no such store: %s", storeName), false)
 	}
@@ -671,15 +668,12 @@ func parsePath(path string) (storeName string, subpath string, err error) {
 // identical across nodes such that chunks from different sources fit together. If the output for a
 // given format changes (at the byte level), the snapshot format must be bumped - see
 // TestMultistoreSnapshot_Checksum test.
-func (rs *Store) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, error) {
-	if format != snapshottypes.CurrentFormat {
-		return nil, sdkerrors.Wrapf(snapshottypes.ErrUnknownFormat, "format %v", format)
-	}
+func (rs *Store) Snapshot(height uint64, protoWriter protoio.Writer) error {
 	if height == 0 {
-		return nil, sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0")
+		return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot snapshot height 0")
 	}
 	if height > uint64(rs.LastCommitID().Version) {
-		return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height)
+		return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot snapshot future height %v", height)
 	}
 
 	// Collect stores to snapshot (only IAVL stores are supported)
@@ -696,7 +690,7 @@ func (rs *Store) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, e
 			// Non-persisted stores shouldn't be snapshotted
 			continue
 		default:
-			return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic,
+			return sdkerrors.Wrapf(sdkerrors.ErrLogic,
 				"don't know how to snapshot store %q of type %T", key.Name(), store)
 		}
 	}
@@ -704,160 +698,97 @@ func (rs *Store) Snapshot(height uint64, format uint32) (<-chan io.ReadCloser, e
 		return strings.Compare(stores[i].name, stores[j].name) == -1
 	})
 
-	// Spawn goroutine to generate snapshot chunks and pass their io.ReadClosers through a channel
-	ch := make(chan io.ReadCloser)
-	go func() {
-		// Set up a stream pipeline to serialize snapshot nodes:
-		// ExportNode -> delimited Protobuf -> zlib -> buffer -> chunkWriter -> chan io.ReadCloser
-		chunkWriter := snapshots.NewChunkWriter(ch, snapshotChunkSize)
-		defer chunkWriter.Close()
-		bufWriter := bufio.NewWriterSize(chunkWriter, snapshotBufferSize)
-		defer func() {
-			if err := bufWriter.Flush(); err != nil {
-				chunkWriter.CloseWithError(err)
-			}
-		}()
-		zWriter, err := zlib.NewWriterLevel(bufWriter, 7)
+	// Export each IAVL store. Stores are serialized as a stream of SnapshotItem Protobuf
+	// messages. The first item contains a SnapshotStore with store metadata (i.e. name),
+	// and the following messages contain a SnapshotNode (i.e. an ExportNode). Store changes
+	// are demarcated by new SnapshotStore items.
+	for _, store := range stores {
+		exporter, err := store.Export(int64(height))
 		if err != nil {
-			chunkWriter.CloseWithError(sdkerrors.Wrap(err, "zlib failure"))
-			return
+			return err
+		}
+		defer exporter.Close()
+		err = protoWriter.WriteMsg(&snapshottypes.SnapshotItem{
+			Item: &snapshottypes.SnapshotItem_Store{
+				Store: &snapshottypes.SnapshotStoreItem{
+					Name: store.name,
+				},
+			},
+		})
+		if err != nil {
+			return err
 		}
-		defer func() {
-			if err := zWriter.Close(); err != nil {
-				chunkWriter.CloseWithError(err)
-			}
-		}()
-		protoWriter := protoio.NewDelimitedWriter(zWriter)
-		defer func() {
-			if err := protoWriter.Close(); err != nil {
-				chunkWriter.CloseWithError(err)
-			}
-		}()
 
-		// Export each IAVL store. Stores are serialized as a stream of SnapshotItem Protobuf
-		// messages. The first item contains a SnapshotStore with store metadata (i.e. name),
-		// and the following messages contain a SnapshotNode (i.e. an ExportNode). Store changes
-		// are demarcated by new SnapshotStore items.
-		for _, store := range stores {
-			exporter, err := store.Export(int64(height))
-			if err != nil {
-				chunkWriter.CloseWithError(err)
-				return
+		for {
+			node, err := exporter.Next()
+			if err == iavltree.ExportDone {
+				break
+			} else if err != nil {
+				return err
 			}
-			defer exporter.Close()
-			err = protoWriter.WriteMsg(&types.SnapshotItem{
-				Item: &types.SnapshotItem_Store{
-					Store: &types.SnapshotStoreItem{
-						Name: store.name,
+			err = protoWriter.WriteMsg(&snapshottypes.SnapshotItem{
+				Item: &snapshottypes.SnapshotItem_IAVL{
+					IAVL: &snapshottypes.SnapshotIAVLItem{
+						Key:     node.Key,
+						Value:   node.Value,
+						Height:  int32(node.Height),
+						Version: node.Version,
 					},
 				},
 			})
 			if err != nil {
-				chunkWriter.CloseWithError(err)
-				return
+				return err
 			}
-
-			for {
-				node, err := exporter.Next()
-				if err == iavltree.ExportDone {
-					break
-				} else if err != nil {
-					chunkWriter.CloseWithError(err)
-					return
-				}
-				err = protoWriter.WriteMsg(&types.SnapshotItem{
-					Item: &types.SnapshotItem_IAVL{
-						IAVL: &types.SnapshotIAVLItem{
-							Key:     node.Key,
-							Value:   node.Value,
-							Height:  int32(node.Height),
-							Version: node.Version,
-						},
-					},
-				})
-				if err != nil {
-					chunkWriter.CloseWithError(err)
-					return
-				}
-			}
-			exporter.Close()
 		}
-	}()
+		exporter.Close()
+	}
 
-	return ch, nil
+	return nil
 }
 
 // Restore implements snapshottypes.Snapshotter.
+// returns next snapshot item and error.
 func (rs *Store) Restore(
-	height uint64, format uint32, chunks <-chan io.ReadCloser, ready chan<- struct{},
-) error {
-	if format != snapshottypes.CurrentFormat {
-		return sdkerrors.Wrapf(snapshottypes.ErrUnknownFormat, "format %v", format)
-	}
-	if height == 0 {
-		return sdkerrors.Wrap(sdkerrors.ErrLogic, "cannot restore snapshot at height 0")
-	}
-	if height > uint64(math.MaxInt64) {
-		return sdkerrors.Wrapf(snapshottypes.ErrInvalidMetadata,
-			"snapshot height %v cannot exceed %v", height, int64(math.MaxInt64))
-	}
-
-	// Signal readiness. Must be done before the readers below are set up, since the zlib
-	// reader reads from the stream on initialization, potentially causing deadlocks.
-	if ready != nil {
-		close(ready)
-	}
-
-	// Set up a restore stream pipeline
-	// chan io.ReadCloser -> chunkReader -> zlib -> delimited Protobuf -> ExportNode
-	chunkReader := snapshots.NewChunkReader(chunks)
-	defer chunkReader.Close()
-	zReader, err := zlib.NewReader(chunkReader)
-	if err != nil {
-		return sdkerrors.Wrap(err, "zlib failure")
-	}
-	defer zReader.Close()
-	protoReader := protoio.NewDelimitedReader(zReader, snapshotMaxItemSize)
-	defer protoReader.Close()
-
+	height uint64, format uint32, protoReader protoio.Reader,
+) (snapshottypes.SnapshotItem, error) {
 	// Import nodes into stores. The first item is expected to be a SnapshotItem containing
 	// a SnapshotStoreItem, telling us which store to import into. The following items will contain
 	// SnapshotNodeItem (i.e. ExportNode) until we reach the next SnapshotStoreItem or EOF.
 	var importer *iavltree.Importer
 	for {
-		item := &types.SnapshotItem{}
-		err := protoReader.ReadMsg(item)
+		snapshotItem := &snapshottypes.SnapshotItem{}
+		err := protoReader.ReadMsg(snapshotItem)
 		if err == io.EOF {
 			break
 		} else if err != nil {
-			return sdkerrors.Wrap(err, "invalid protobuf message")
+			return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "invalid protobuf message")
 		}
 
-		switch item := item.Item.(type) {
-		case *types.SnapshotItem_Store:
+		switch item := snapshotItem.Item.(type) {
+		case *snapshottypes.SnapshotItem_Store:
 			if importer != nil {
 				err = importer.Commit()
 				if err != nil {
-					return sdkerrors.Wrap(err, "IAVL commit failed")
+					return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "IAVL commit failed")
 				}
 				importer.Close()
 			}
-			store, ok := rs.getStoreByName(item.Store.Name).(*iavl.Store)
+			store, ok := rs.GetStoreByName(item.Store.Name).(*iavl.Store)
 			if !ok || store == nil {
-				return sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot import into non-IAVL store %q", item.Store.Name)
+				return snapshottypes.SnapshotItem{}, sdkerrors.Wrapf(sdkerrors.ErrLogic, "cannot import into non-IAVL store %q", item.Store.Name)
 			}
 			importer, err = store.Import(int64(height))
 			if err != nil {
-				return sdkerrors.Wrap(err, "import failed")
+				return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "import failed")
 			}
 			defer importer.Close()
 
-		case *types.SnapshotItem_IAVL:
+		case *snapshottypes.SnapshotItem_IAVL:
 			if importer == nil {
-				return sdkerrors.Wrap(sdkerrors.ErrLogic, "received IAVL node item before store item")
+				return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(sdkerrors.ErrLogic, "received IAVL node item before store item")
 			}
 			if item.IAVL.Height > math.MaxInt8 {
-				return sdkerrors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",
+				return snapshottypes.SnapshotItem{}, sdkerrors.Wrapf(sdkerrors.ErrLogic, "node height %v cannot exceed %v",
 					item.IAVL.Height, math.MaxInt8)
 			}
 			node := &iavltree.ExportNode{
@@ -876,24 +807,25 @@ func (rs *Store) Restore(
 			}
 			err := importer.Add(node)
 			if err != nil {
-				return sdkerrors.Wrap(err, "IAVL node import failed")
+				return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "IAVL node import failed")
 			}
 
 		default:
-			return sdkerrors.Wrapf(sdkerrors.ErrLogic, "unknown snapshot item %T", item)
+			// pass back the unrecognized item.
+			return *snapshotItem, nil
 		}
 	}
 
 	if importer != nil {
 		err := importer.Commit()
 		if err != nil {
-			return sdkerrors.Wrap(err, "IAVL commit failed")
+			return snapshottypes.SnapshotItem{}, sdkerrors.Wrap(err, "IAVL commit failed")
 		}
 		importer.Close()
 	}
 
 	flushMetadata(rs.db, int64(height), rs.buildCommitInfo(int64(height)), []int64{})
-	return rs.LoadLatestVersion()
+	return snapshottypes.SnapshotItem{}, rs.LoadLatestVersion()
 }
 
 func (rs *Store) loadCommitStoreFromParams(key types.StoreKey, id types.CommitID, params storeParams) (types.CommitKVStore, error) {

store/rootmulti/store_test.go

@@ -2,24 +2,16 @@ package rootmulti
 
 import (
 	"bytes"
-	"crypto/sha256"
-	"encoding/binary"
-	"encoding/hex"
-	"errors"
 	"fmt"
-	"io"
-	"math/rand"
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	abci "github.com/tendermint/tendermint/abci/types"
 	dbm "github.com/tendermint/tm-db"
 
 	"github.com/cosmos/cosmos-sdk/codec"
 	codecTypes "github.com/cosmos/cosmos-sdk/codec/types"
-	snapshottypes "github.com/cosmos/cosmos-sdk/snapshots/types"
 	"github.com/cosmos/cosmos-sdk/store/cachemulti"
 	"github.com/cosmos/cosmos-sdk/store/iavl"
 	sdkmaps "github.com/cosmos/cosmos-sdk/store/internal/maps"
@@ -86,7 +78,7 @@ func TestCacheMultiStoreWithVersion(t *testing.T) {
 
 	k, v := []byte("wind"), []byte("blows")
 
-	store1 := ms.getStoreByName("store1").(types.KVStore)
+	store1 := ms.GetStoreByName("store1").(types.KVStore)
 	store1.Set(k, v)
 
 	cID := ms.Commit()
@@ -123,7 +115,7 @@ func TestHashStableWithEmptyCommit(t *testing.T) {
 
 	k, v := []byte("wind"), []byte("blows")
 
-	store1 := ms.getStoreByName("store1").(types.KVStore)
+	store1 := ms.GetStoreByName("store1").(types.KVStore)
 	store1.Set(k, v)
 
 	cID := ms.Commit()
@@ -147,11 +139,11 @@ func TestMultistoreCommitLoad(t *testing.T) {
 	checkStore(t, store, commitID, commitID)
 
 	// Make sure we can get stores by name.
-	s1 := store.getStoreByName("store1")
+	s1 := store.GetStoreByName("store1")
 	require.NotNil(t, s1)
-	s3 := store.getStoreByName("store3")
+	s3 := store.GetStoreByName("store3")
 	require.NotNil(t, s3)
-	s77 := store.getStoreByName("store77")
+	s77 := store.GetStoreByName("store77")
 	require.Nil(t, s77)
 
 	// Make a few commits and check them.
@@ -191,21 +183,21 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
 
 	// write some data in all stores
 	k1, v1 := []byte("first"), []byte("store")
-	s1, _ := store.getStoreByName("store1").(types.KVStore)
+	s1, _ := store.GetStoreByName("store1").(types.KVStore)
 	require.NotNil(t, s1)
 	s1.Set(k1, v1)
 
 	k2, v2 := []byte("second"), []byte("restore")
-	s2, _ := store.getStoreByName("store2").(types.KVStore)
+	s2, _ := store.GetStoreByName("store2").(types.KVStore)
 	require.NotNil(t, s2)
 	s2.Set(k2, v2)
 
 	k3, v3 := []byte("third"), []byte("dropped")
-	s3, _ := store.getStoreByName("store3").(types.KVStore)
+	s3, _ := store.GetStoreByName("store3").(types.KVStore)
 	require.NotNil(t, s3)
 	s3.Set(k3, v3)
 
-	s4, _ := store.getStoreByName("store4").(types.KVStore)
+	s4, _ := store.GetStoreByName("store4").(types.KVStore)
 	require.Nil(t, s4)
 
 	// do one commit
@@ -228,7 +220,7 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
 	checkStore(t, store, commitID, commitID)
 
 	// let's query data to see it was saved properly
-	s2, _ = store.getStoreByName("store2").(types.KVStore)
+	s2, _ = store.GetStoreByName("store2").(types.KVStore)
 	require.NotNil(t, s2)
 	require.Equal(t, v2, s2.Get(k2))
 
@@ -238,17 +230,17 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
 	require.Nil(t, err)
 
 	// s1 was not changed
-	s1, _ = restore.getStoreByName("store1").(types.KVStore)
+	s1, _ = restore.GetStoreByName("store1").(types.KVStore)
 	require.NotNil(t, s1)
 	require.Equal(t, v1, s1.Get(k1))
 
 	// store3 is mounted, but data deleted are gone
-	s3, _ = restore.getStoreByName("store3").(types.KVStore)
+	s3, _ = restore.GetStoreByName("store3").(types.KVStore)
 	require.NotNil(t, s3)
 	require.Nil(t, s3.Get(k3)) // data was deleted
 
 	// store4 is mounted, with empty data
-	s4, _ = restore.getStoreByName("store4").(types.KVStore)
+	s4, _ = restore.GetStoreByName("store4").(types.KVStore)
 	require.NotNil(t, s4)
 
 	iterator := s4.Iterator(nil, nil)
@@ -266,11 +258,11 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
 	s4.Set(k4, v4)
 
 	// store2 is no longer mounted
-	st2 := restore.getStoreByName("store2")
+	st2 := restore.GetStoreByName("store2")
 	require.Nil(t, st2)
 
 	// restore2 has the old data
-	rs2, _ := restore.getStoreByName("restore2").(types.KVStore)
+	rs2, _ := restore.GetStoreByName("restore2").(types.KVStore)
 	require.NotNil(t, rs2)
 	require.Equal(t, v2, rs2.Get(k2))
 
@@ -284,15 +276,15 @@ func TestMultistoreLoadWithUpgrade(t *testing.T) {
 	require.Equal(t, migratedID, reload.LastCommitID())
 
 	// query this new store
-	rl1, _ := reload.getStoreByName("store1").(types.KVStore)
+	rl1, _ := reload.GetStoreByName("store1").(types.KVStore)
 	require.NotNil(t, rl1)
 	require.Equal(t, v1, rl1.Get(k1))
 
-	rl2, _ := reload.getStoreByName("restore2").(types.KVStore)
+	rl2, _ := reload.GetStoreByName("restore2").(types.KVStore)
 	require.NotNil(t, rl2)
 	require.Equal(t, v2, rl2.Get(k2))
 
-	rl4, _ := reload.getStoreByName("store4").(types.KVStore)
+	rl4, _ := reload.GetStoreByName("store4").(types.KVStore)
 	require.NotNil(t, rl4)
 	require.Equal(t, v4, rl4.Get(k4))
 
@@ -343,15 +335,15 @@ func TestMultiStoreRestart(t *testing.T) {
 
 	for i := 1; i < 3; i++ {
 		// Set and commit data in one store.
-		store1 := multi.getStoreByName("store1").(types.KVStore)
+		store1 := multi.GetStoreByName("store1").(types.KVStore)
 		store1.Set([]byte(k), []byte(fmt.Sprintf("%s:%d", v, i)))
 
 		// ... and another.
-		store2 := multi.getStoreByName("store2").(types.KVStore)
+		store2 := multi.GetStoreByName("store2").(types.KVStore)
 		store2.Set([]byte(k2), []byte(fmt.Sprintf("%s:%d", v2, i)))
 
 		// ... and another.
-		store3 := multi.getStoreByName("store3").(types.KVStore)
+		store3 := multi.GetStoreByName("store3").(types.KVStore)
 		store3.Set([]byte(k3), []byte(fmt.Sprintf("%s:%d", v3, i)))
 
 		multi.Commit()
@@ -362,11 +354,11 @@ func TestMultiStoreRestart(t *testing.T) {
 	}
 
 	// Set and commit data in one store.
-	store1 := multi.getStoreByName("store1").(types.KVStore)
+	store1 := multi.GetStoreByName("store1").(types.KVStore)
 	store1.Set([]byte(k), []byte(fmt.Sprintf("%s:%d", v, 3)))
 
 	// ... and another.
-	store2 := multi.getStoreByName("store2").(types.KVStore)
+	store2 := multi.GetStoreByName("store2").(types.KVStore)
 	store2.Set([]byte(k2), []byte(fmt.Sprintf("%s:%d", v2, 3)))
 
 	multi.Commit()
@@ -376,7 +368,7 @@ func TestMultiStoreRestart(t *testing.T) {
 	require.NotEqual(t, initCid, flushedCinfo, "CID is different after flush to disk")
 
 	// ... and another.
-	store3 := multi.getStoreByName("store3").(types.KVStore)
+	store3 := multi.GetStoreByName("store3").(types.KVStore)
 	store3.Set([]byte(k3), []byte(fmt.Sprintf("%s:%d", v3, 3)))
 
 	multi.Commit()
@@ -393,16 +385,16 @@ func TestMultiStoreRestart(t *testing.T) {
 	require.Equal(t, int64(4), reloadedCid.Version, "Reloaded CID is not the same as last flushed CID")
 
 	// Check that store1 and store2 retained date from 3rd commit
-	store1 = multi.getStoreByName("store1").(types.KVStore)
+	store1 = multi.GetStoreByName("store1").(types.KVStore)
 	val := store1.Get([]byte(k))
 	require.Equal(t, []byte(fmt.Sprintf("%s:%d", v, 3)), val, "Reloaded value not the same as last flushed value")
 
-	store2 = multi.getStoreByName("store2").(types.KVStore)
+	store2 = multi.GetStoreByName("store2").(types.KVStore)
 	val2 := store2.Get([]byte(k2))
 	require.Equal(t, []byte(fmt.Sprintf("%s:%d", v2, 3)), val2, "Reloaded value not the same as last flushed value")
 
 	// Check that store3 still has data from last commit even though update happened on 2nd commit
-	store3 = multi.getStoreByName("store3").(types.KVStore)
+	store3 = multi.GetStoreByName("store3").(types.KVStore)
 	val3 := store3.Get([]byte(k3))
 	require.Equal(t, []byte(fmt.Sprintf("%s:%d", v3, 3)), val3, "Reloaded value not the same as last flushed value")
 }
@@ -420,15 +412,15 @@ func TestMultiStoreQuery(t *testing.T) {
 	cid := multi.Commit()
 
 	// Make sure we can get by name.
-	garbage := multi.getStoreByName("bad-name")
+	garbage := multi.GetStoreByName("bad-name")
 	require.Nil(t, garbage)
 
 	// Set and commit data in one store.
-	store1 := multi.getStoreByName("store1").(types.KVStore)
+	store1 := multi.GetStoreByName("store1").(types.KVStore)
 	store1.Set(k, v)
 
 	// ... and another.
-	store2 := multi.getStoreByName("store2").(types.KVStore)
+	store2 := multi.GetStoreByName("store2").(types.KVStore)
 	store2.Set(k2, v2)
 
 	// Commit the multistore.
@@ -551,121 +543,6 @@ func TestMultiStore_PruningRestart(t *testing.T) {
 	}
 }
 
-func TestMultistoreSnapshot_Checksum(t *testing.T) {
-	// Chunks from different nodes must fit together, so all nodes must produce identical chunks.
-	// This checksum test makes sure that the byte stream remains identical. If the test fails
-	// without having changed the data (e.g. because the Protobuf or zlib encoding changes),
-	// snapshottypes.CurrentFormat must be bumped.
-	store := newMultiStoreWithGeneratedData(dbm.NewMemDB(), 5, 10000)
-	version := uint64(store.LastCommitID().Version)
-
-	testcases := []struct {
-		format      uint32
-		chunkHashes []string
-	}{
-		{1, []string{
-			"503e5b51b657055b77e88169fadae543619368744ad15f1de0736c0a20482f24",
-			"e1a0daaa738eeb43e778aefd2805e3dd720798288a410b06da4b8459c4d8f72e",
-			"aa048b4ee0f484965d7b3b06822cf0772cdcaad02f3b1b9055e69f2cb365ef3c",
-			"7921eaa3ed4921341e504d9308a9877986a879fe216a099c86e8db66fcba4c63",
-			"a4a864e6c02c9fca5837ec80dc84f650b25276ed7e4820cf7516ced9f9901b86",
-			"ca2879ac6e7205d257440131ba7e72bef784cd61642e32b847729e543c1928b9",
-		}},
-	}
-	for _, tc := range testcases {
-		tc := tc
-		t.Run(fmt.Sprintf("Format %v", tc.format), func(t *testing.T) {
-			chunks, err := store.Snapshot(version, tc.format)
-			require.NoError(t, err)
-			hashes := []string{}
-			hasher := sha256.New()
-			for chunk := range chunks {
-				hasher.Reset()
-				_, err := io.Copy(hasher, chunk)
-				require.NoError(t, err)
-				hashes = append(hashes, hex.EncodeToString(hasher.Sum(nil)))
-			}
-			assert.Equal(t, tc.chunkHashes, hashes,
-				"Snapshot output for format %v has changed", tc.format)
-		})
-	}
-}
-
-func TestMultistoreSnapshot_Errors(t *testing.T) {
-	store := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB())
-
-	testcases := map[string]struct {
-		height     uint64
-		format     uint32
-		expectType error
-	}{
-		"0 height":       {0, snapshottypes.CurrentFormat, nil},
-		"0 format":       {1, 0, snapshottypes.ErrUnknownFormat},
-		"unknown height": {9, snapshottypes.CurrentFormat, nil},
-		"unknown format": {1, 9, snapshottypes.ErrUnknownFormat},
-	}
-	for name, tc := range testcases {
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			_, err := store.Snapshot(tc.height, tc.format)
-			require.Error(t, err)
-			if tc.expectType != nil {
-				assert.True(t, errors.Is(err, tc.expectType))
-			}
-		})
-	}
-}
-
-func TestMultistoreRestore_Errors(t *testing.T) {
-	store := newMultiStoreWithMixedMounts(dbm.NewMemDB())
-
-	testcases := map[string]struct {
-		height     uint64
-		format     uint32
-		expectType error
-	}{
-		"0 height":       {0, snapshottypes.CurrentFormat, nil},
-		"0 format":       {1, 0, snapshottypes.ErrUnknownFormat},
-		"unknown format": {1, 9, snapshottypes.ErrUnknownFormat},
-	}
-	for name, tc := range testcases {
-		tc := tc
-		t.Run(name, func(t *testing.T) {
-			err := store.Restore(tc.height, tc.format, nil, nil)
-			require.Error(t, err)
-			if tc.expectType != nil {
-				assert.True(t, errors.Is(err, tc.expectType))
-			}
-		})
-	}
-}
-
-func TestMultistoreSnapshotRestore(t *testing.T) {
-	source := newMultiStoreWithMixedMountsAndBasicData(dbm.NewMemDB())
-	target := newMultiStoreWithMixedMounts(dbm.NewMemDB())
-	version := uint64(source.LastCommitID().Version)
-	require.EqualValues(t, 3, version)
-
-	chunks, err := source.Snapshot(version, snapshottypes.CurrentFormat)
-	require.NoError(t, err)
-	ready := make(chan struct{})
-	err = target.Restore(version, snapshottypes.CurrentFormat, chunks, ready)
-	require.NoError(t, err)
-	assert.EqualValues(t, struct{}{}, <-ready)
-
-	assert.Equal(t, source.LastCommitID(), target.LastCommitID())
-	for key, sourceStore := range source.stores {
-		targetStore := target.getStoreByName(key.Name()).(types.CommitKVStore)
-		switch sourceStore.GetStoreType() {
-		case types.StoreTypeTransient:
-			assert.False(t, targetStore.Iterator(nil, nil).Valid(),
-				"transient store %v not empty", key.Name())
-		default:
-			assertStoresEqual(t, sourceStore, targetStore, "store %q not equal", key.Name())
-		}
-	}
-}
-
 func TestSetInitialVersion(t *testing.T) {
 	db := dbm.NewMemDB()
 	multi := newMultiStoreWithMounts(db, types.PruneNothing)
@@ -853,79 +730,6 @@ func TestTraceConcurrency(t *testing.T) {
 	stopW <- struct{}{}
 }
 
-func BenchmarkMultistoreSnapshot100K(b *testing.B) {
-	benchmarkMultistoreSnapshot(b, 10, 10000)
-}
-
-func BenchmarkMultistoreSnapshot1M(b *testing.B) {
-	benchmarkMultistoreSnapshot(b, 10, 100000)
-}
-
-func BenchmarkMultistoreSnapshotRestore100K(b *testing.B) {
-	benchmarkMultistoreSnapshotRestore(b, 10, 10000)
-}
-
-func BenchmarkMultistoreSnapshotRestore1M(b *testing.B) {
-	benchmarkMultistoreSnapshotRestore(b, 10, 100000)
-}
-
-func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) {
-	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
-
-	b.ReportAllocs()
-	b.StopTimer()
-	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
-	version := source.LastCommitID().Version
-	require.EqualValues(b, 1, version)
-	b.StartTimer()
-
-	for i := 0; i < b.N; i++ {
-		target := NewStore(dbm.NewMemDB())
-		for key := range source.stores {
-			target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
-		}
-		err := target.LoadLatestVersion()
-		require.NoError(b, err)
-		require.EqualValues(b, 0, target.LastCommitID().Version)
-
-		chunks, err := source.Snapshot(uint64(version), snapshottypes.CurrentFormat)
-		require.NoError(b, err)
-		for reader := range chunks {
-			_, err := io.Copy(io.Discard, reader)
-			require.NoError(b, err)
-			err = reader.Close()
-			require.NoError(b, err)
-		}
-	}
-}
-
-func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys uint64) {
-	b.Skip("Noisy with slow setup time, please see https://github.com/cosmos/cosmos-sdk/issues/8855.")
-
-	b.ReportAllocs()
-	b.StopTimer()
-	source := newMultiStoreWithGeneratedData(dbm.NewMemDB(), stores, storeKeys)
-	version := uint64(source.LastCommitID().Version)
-	require.EqualValues(b, 1, version)
-	b.StartTimer()
-
-	for i := 0; i < b.N; i++ {
-		target := NewStore(dbm.NewMemDB())
-		for key := range source.stores {
-			target.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
-		}
-		err := target.LoadLatestVersion()
-		require.NoError(b, err)
-		require.EqualValues(b, 0, target.LastCommitID().Version)
-
-		chunks, err := source.Snapshot(version, snapshottypes.CurrentFormat)
-		require.NoError(b, err)
-		err = target.Restore(version, snapshottypes.CurrentFormat, chunks, nil)
-		require.NoError(b, err)
-		require.Equal(b, source.LastCommitID(), target.LastCommitID())
-	}
-}
-
 //-----------------------------------------------------------------------
 // utils
 
@@ -946,75 +750,6 @@ func newMultiStoreWithMounts(db dbm.DB, pruningOpts types.PruningOptions) *Store
 	return store
 }
 
-func newMultiStoreWithMixedMounts(db dbm.DB) *Store {
-	store := NewStore(db)
-	store.MountStoreWithDB(types.NewKVStoreKey("iavl1"), types.StoreTypeIAVL, nil)
-	store.MountStoreWithDB(types.NewKVStoreKey("iavl2"), types.StoreTypeIAVL, nil)
-	store.MountStoreWithDB(types.NewKVStoreKey("iavl3"), types.StoreTypeIAVL, nil)
-	store.MountStoreWithDB(types.NewTransientStoreKey("trans1"), types.StoreTypeTransient, nil)
-	store.LoadLatestVersion()
-
-	return store
-}
-
-func newMultiStoreWithMixedMountsAndBasicData(db dbm.DB) *Store {
-	store := newMultiStoreWithMixedMounts(db)
-	store1 := store.getStoreByName("iavl1").(types.CommitKVStore)
-	store2 := store.getStoreByName("iavl2").(types.CommitKVStore)
-	trans1 := store.getStoreByName("trans1").(types.KVStore)
-
-	store1.Set([]byte("a"), []byte{1})
-	store1.Set([]byte("b"), []byte{1})
-	store2.Set([]byte("X"), []byte{255})
-	store2.Set([]byte("A"), []byte{101})
-	trans1.Set([]byte("x1"), []byte{91})
-	store.Commit()
-
-	store1.Set([]byte("b"), []byte{2})
-	store1.Set([]byte("c"), []byte{3})
-	store2.Set([]byte("B"), []byte{102})
-	store.Commit()
-
-	store2.Set([]byte("C"), []byte{103})
-	store2.Delete([]byte("X"))
-	trans1.Set([]byte("x2"), []byte{92})
-	store.Commit()
-
-	return store
-}
-
-func newMultiStoreWithGeneratedData(db dbm.DB, stores uint8, storeKeys uint64) *Store {
-	multiStore := NewStore(db)
-	r := rand.New(rand.NewSource(49872768940)) // Fixed seed for deterministic tests
-
-	keys := []*types.KVStoreKey{}
-	for i := uint8(0); i < stores; i++ {
-		key := types.NewKVStoreKey(fmt.Sprintf("store%v", i))
-		multiStore.MountStoreWithDB(key, types.StoreTypeIAVL, nil)
-		keys = append(keys, key)
-	}
-	multiStore.LoadLatestVersion()
-
-	for _, key := range keys {
-		store := multiStore.stores[key].(*iavl.Store)
-		for i := uint64(0); i < storeKeys; i++ {
-			k := make([]byte, 8)
-			v := make([]byte, 1024)
-			binary.BigEndian.PutUint64(k, i)
-			_, err := r.Read(v)
-			if err != nil {
-				panic(err)
-			}
-			store.Set(k, v)
-		}
-	}
-
-	multiStore.Commit()
-	multiStore.LoadLatestVersion()
-
-	return multiStore
-}
-
 func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions) (*Store, *types.StoreUpgrades) {
 	store := NewStore(db)
 	store.pruningOpts = pruningOpts
@@ -1036,25 +771,6 @@ func newMultiStoreWithModifiedMounts(db dbm.DB, pruningOpts types.PruningOptions
 	return store, upgrades
 }
 
-func assertStoresEqual(t *testing.T, expect, actual types.CommitKVStore, msgAndArgs ...interface{}) {
-	assert.Equal(t, expect.LastCommitID(), actual.LastCommitID())
-	expectIter := expect.Iterator(nil, nil)
-	expectMap := map[string][]byte{}
-	for ; expectIter.Valid(); expectIter.Next() {
-		expectMap[string(expectIter.Key())] = expectIter.Value()
-	}
-	require.NoError(t, expectIter.Error())
-
-	actualIter := expect.Iterator(nil, nil)
-	actualMap := map[string][]byte{}
-	for ; actualIter.Valid(); actualIter.Next() {
-		actualMap[string(actualIter.Key())] = actualIter.Value()
-	}
-	require.NoError(t, actualIter.Error())
-
-	assert.Equal(t, expectMap, actualMap, msgAndArgs...)
-}
-
 func checkStore(t *testing.T, store *Store, expect, got types.CommitID) {
 	require.Equal(t, expect, got)
 	require.Equal(t, expect, store.LastCommitID())