From b314b851e4300e30a22c9e3fd9b5e2602e7c4e92 Mon Sep 17 00:00:00 2001 From: Dmitry Shulyak Date: Thu, 26 Dec 2019 17:33:34 +0200 Subject: [PATCH] Merge PR #5435: Added iterator that allows to read only requested values --- CHANGELOG.md | 3 + store/types/iterator.go | 61 ++++++++++++++++++ store/types/iterator_test.go | 119 +++++++++++++++++++++++++++++++++++ types/store.go | 12 ++++ 4 files changed, 195 insertions(+) create mode 100644 store/types/iterator.go create mode 100644 store/types/iterator_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e7b2bd071e..ef0b35e520 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,7 @@ increased significantly due to modular `AnteHandler` support. Increase GasLimit ### Features +* (store) [\#5435](https://github.com/cosmos/cosmos-sdk/pull/5435) New iterator for paginated requests. Iterator limits DB reads to the range of the requested page. * (x/evidence) [\#5240](https://github.com/cosmos/cosmos-sdk/pull/5240) Initial implementation of the `x/evidence` module. * (cli) [\#5212](https://github.com/cosmos/cosmos-sdk/issues/5212) The `q gov proposals` command now supports pagination. * (store) [\#4724](https://github.com/cosmos/cosmos-sdk/issues/4724) Multistore supports substore migrations upon load. New `rootmulti.Store.LoadLatestVersionAndUpgrade` method in @@ -2804,3 +2805,5 @@ BUG FIXES: [v0.37.1]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.1 [v0.37.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.37.0 [v0.36.0]: https://github.com/cosmos/cosmos-sdk/releases/tag/v0.36.0 + + diff --git a/store/types/iterator.go b/store/types/iterator.go new file mode 100644 index 0000000000..cfce4124e3 --- /dev/null +++ b/store/types/iterator.go @@ -0,0 +1,61 @@ +package types + +import ( + "fmt" +) + +// KVStorePrefixIteratorPaginated returns iterator over items in the selected page. +// Items iterated and skipped in ascending order. +func KVStorePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator { + pi := &PaginatedIterator{ + Iterator: KVStorePrefixIterator(kvs, prefix), + page: page, + limit: limit, + } + pi.skip() + return pi +} + +// KVStoreReversePrefixIteratorPaginated returns iterator over items in the selected page. +// Items iterated and skipped in descending order. +func KVStoreReversePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator { + pi := &PaginatedIterator{ + Iterator: KVStoreReversePrefixIterator(kvs, prefix), + page: page, + limit: limit, + } + pi.skip() + return pi +} + +// PaginatedIterator is a wrapper around Iterator that iterates over values starting for given page and limit. +type PaginatedIterator struct { + Iterator + + page, limit uint // provided during initialization + iterated uint // incremented in a call to Next + +} + +func (pi *PaginatedIterator) skip() { + for i := (pi.page - 1) * pi.limit; i > 0 && pi.Iterator.Valid(); i-- { + pi.Iterator.Next() + } +} + +// Next will panic after limit is reached. +func (pi *PaginatedIterator) Next() { + if !pi.Valid() { + panic(fmt.Sprintf("PaginatedIterator reached limit %d", pi.limit)) + } + pi.Iterator.Next() + pi.iterated++ +} + +// Valid if below limit and underlying iterator is valid. +func (pi *PaginatedIterator) Valid() bool { + if pi.iterated >= pi.limit { + return false + } + return pi.Iterator.Valid() +} diff --git a/store/types/iterator_test.go b/store/types/iterator_test.go new file mode 100644 index 0000000000..e081de80a3 --- /dev/null +++ b/store/types/iterator_test.go @@ -0,0 +1,119 @@ +package types_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + + "github.com/cosmos/cosmos-sdk/store/iavl" + "github.com/cosmos/cosmos-sdk/store/types" +) + +func newMemTestKVStore(t *testing.T) types.KVStore { + db := dbm.NewMemDB() + store, err := iavl.LoadStore(db, types.CommitID{}, types.PruneNothing, false) + require.NoError(t, err) + return store +} + +func TestPaginatedIterator(t *testing.T) { + kvs := newMemTestKVStore(t) + total := 10 + lth := total - 1 + asc := make([][]byte, total) + desc := make([][]byte, total) + // store returns values in lexicographic order (or reverse lex order) + for i := 0; i < total; i++ { + key := []byte{byte(i)} + kvs.Set(key, key) + asc[i] = key + desc[lth-i] = key + } + type testCase struct { + desc string + page, limit uint + result [][]byte + reverse bool + } + for _, tc := range []testCase{ + { + desc: "FirstChunk", + page: 1, + limit: 4, + result: asc[:4], + }, + { + desc: "SecondChunk", + page: 2, + limit: 4, + result: asc[4:8], + }, + { + desc: "ThirdChunkHalf", + page: 3, + limit: 4, + result: asc[8:], + }, + { + desc: "OverLimit", + page: 10, + limit: 10, + result: [][]byte{}, + }, + { + desc: "ZeroLimit", + page: 1, + result: [][]byte{}, + }, + { + desc: "ReverseFirstChunk", + page: 1, + limit: 6, + result: desc[:6], + reverse: true, + }, + { + desc: "ReverseSecondChunk", + page: 2, + limit: 6, + result: desc[6:], + reverse: true, + }, + } { + tc := tc + t.Run(tc.desc, func(t *testing.T) { + var iter types.Iterator + if tc.reverse { + iter = types.KVStoreReversePrefixIteratorPaginated(kvs, nil, tc.page, tc.limit) + } else { + iter = types.KVStorePrefixIteratorPaginated(kvs, nil, tc.page, tc.limit) + } + defer iter.Close() + + result := [][]byte{} + for ; iter.Valid(); iter.Next() { + result = append(result, iter.Key()) + } + + require.Equal(t, tc.result, result) + require.False(t, iter.Valid()) + }) + } +} + +func TestPaginatedIteratorPanicIfInvalid(t *testing.T) { + kvs := newMemTestKVStore(t) + + iter := types.KVStorePrefixIteratorPaginated(kvs, nil, 1, 1) + defer iter.Close() + require.False(t, iter.Valid()) + require.Panics(t, func() { iter.Next() }) // "iterator is empty" + + kvs.Set([]byte{1}, []byte{}) + + iter = types.KVStorePrefixIteratorPaginated(kvs, nil, 1, 0) + defer iter.Close() + require.False(t, iter.Valid()) + require.Panics(t, func() { iter.Next() }) // "not empty but limit is zero" +} diff --git a/types/store.go b/types/store.go index c34b6d437d..1c50444c0e 100644 --- a/types/store.go +++ b/types/store.go @@ -40,6 +40,18 @@ func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator { return types.KVStoreReversePrefixIterator(kvs, prefix) } +// KVStorePrefixIteratorPaginated returns iterator over items in the selected page. +// Items iterated and skipped in ascending order. +func KVStorePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator { + return types.KVStorePrefixIteratorPaginated(kvs, prefix, page, limit) +} + +// KVStoreReversePrefixIteratorPaginated returns iterator over items in the selected page. +// Items iterated and skipped in descending order. +func KVStoreReversePrefixIteratorPaginated(kvs KVStore, prefix []byte, page, limit uint) Iterator { + return types.KVStorePrefixIteratorPaginated(kvs, prefix, page, limit) +} + // DiffKVStores compares two KVstores and returns all the key/value pairs // that differ from one another. It also skips value comparison for a set of provided prefixes func DiffKVStores(a KVStore, b KVStore, prefixesToSkip [][]byte) (kvAs, kvBs []cmn.KVPair) {