LaconicNetwork/cosmos-sdk
7
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 2 Wiki Activity

feat(collections): implement pagination (#14468)

Browse Source 
Co-authored-by: testinginprod <testinginprod@somewhere.idk>
This commit is contained in:
testinginprod 2023-01-06 12:08:31 +01:00 committed by GitHub
parent 512953cd68
commit f771f20da4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 549 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -39,6 +39,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
### Features
* (query) [#14468](https://github.com/cosmos/cosmos-sdk/pull/14468) Implement pagination for collections.
* (x/bank) [#14045](https://github.com/cosmos/cosmos-sdk/pull/14045) Add CLI command `spendable-balances`, which also accepts the flag `--denom`.
* (x/slashing, x/staking) [#14363](https://github.com/cosmos/cosmos-sdk/pull/14363) Add the infraction a validator commited type as an argument to a `SlashWithInfractionReason` keeper method.
* (client) [#13867](https://github.com/cosmos/cosmos-sdk/pull/13867/) Wire AutoCLI commands with SimApp.

3
collections/CHANGELOG.md
View File

@ -33,4 +33,5 @@ Ref: https://keepachangelog.com/en/1.0.0/
* [#14134](https://github.com/cosmos/cosmos-sdk/pull/14134) Initialise core (Prefix, KeyEncoder, ValueEncoder, Map).
* [#14351](https://github.com/cosmos/cosmos-sdk/pull/14351) Add keyset
* [#14364](https://github.com/cosmos/cosmos-sdk/pull/14364) Add sequence
* [#14364](https://github.com/cosmos/cosmos-sdk/pull/14364) Add sequence
* [#14468](https://github.com/cosmos/cosmos-sdk/pull/14468) Add Map.IterateRaw API.

3
collections/iter.go
View File

@ -2,10 +2,9 @@ package collections
import (
"context"
"cosmossdk.io/core/store"
"errors"
"fmt"
"cosmossdk.io/core/store"
)
// ErrInvalidIterator is returned when an Iterate call resulted in an invalid iterator.

5
collections/iter_test.go
View File

@ -9,8 +9,11 @@ import (
func TestIteratorBasic(t *testing.T) {
sk, ctx := deps()
// safety check to ensure that iteration does not cross prefix boundaries
sk.OpenKVStore(ctx).Set([]byte{0, 0}, []byte("before prefix"))
sk.OpenKVStore(ctx).Set([]byte{2, 1}, []byte("after prefix"))
schemaBuilder := NewSchemaBuilder(sk)
m := NewMap(schemaBuilder, NewPrefix("some super amazing prefix"), "m", StringKey, Uint64Value)
m := NewMap(schemaBuilder, NewPrefix(1), "m", StringKey, Uint64Value)
_, err := schemaBuilder.Build()
require.NoError(t, err)

43
collections/map.go
View File

@ -128,6 +128,49 @@ func (m Map[K, V]) Iterate(ctx context.Context, ranger Ranger[K]) (Iterator[K, V
return iteratorFromRanger(ctx, m, ranger)
}
// IterateRaw iterates over the collection. The iteration range is untyped, it uses raw
// bytes. The resulting Iterator is typed.
// A nil start iterates from the first key contained in the collection.
// A nil end iterates up to the last key contained in the collection.
// A nil start and a nil end iterates over every key contained in the collection.
// TODO(tip): simplify after https://github.com/cosmos/cosmos-sdk/pull/14310 is merged
func (m Map[K, V]) IterateRaw(ctx context.Context, start, end []byte, order Order) (Iterator[K, V], error) {
prefixedStart := append(m.prefix, start...)
var prefixedEnd []byte
if end == nil {
prefixedEnd = prefixEndBytes(m.prefix)
} else {
prefixedEnd = append(m.prefix, end...)
}
s := m.sa(ctx)
var storeIter store.Iterator
switch order {
case OrderAscending:
storeIter = s.Iterator(prefixedStart, prefixedEnd)
case OrderDescending:
storeIter = s.ReverseIterator(prefixedStart, prefixedEnd)
default:
return Iterator[K, V]{}, errOrder
}
if !storeIter.Valid() {
return Iterator[K, V]{}, ErrInvalidIterator
}
return Iterator[K, V]{
kc: m.kc,
vc: m.vc,
iter: storeIter,
prefixLength: len(m.prefix),
}, nil
}
// KeyCodec returns the Map's KeyCodec.
func (m Map[K, V]) KeyCodec() KeyCodec[K] { return m.kc }
// ValueCodec returns the Map's ValueCodec.
func (m Map[K, V]) ValueCodec() ValueCodec[V] { return m.vc }
func encodeKeyWithPrefix[K any](prefix []byte, kc KeyCodec[K], key K) ([]byte, error) {
prefixLen := len(prefix)
// preallocate buffer

35
collections/map_test.go
View File

@ -36,6 +36,41 @@ func TestMap(t *testing.T) {
require.False(t, has)
}
func TestMap_IterateRaw(t *testing.T) {
sk, ctx := deps()
// safety check to ensure prefix boundaries are not crossed
sk.OpenKVStore(ctx).Set([]byte{0x0, 0x0}, []byte("before prefix"))
sk.OpenKVStore(ctx).Set([]byte{0x2, 0x0}, []byte("after prefix"))
sb := NewSchemaBuilder(sk)
m := NewMap(sb, NewPrefix(1), "m", Uint64Key, Uint64Value)
require.NoError(t, m.Set(ctx, 0, 0))
require.NoError(t, m.Set(ctx, 1, 1))
require.NoError(t, m.Set(ctx, 2, 2))
// test non nil end in ascending order
twoBigEndian, err := encodeKeyWithPrefix(nil, Uint64Key, 2)
require.NoError(t, err)
iter, err := m.IterateRaw(ctx, nil, twoBigEndian, OrderAscending)
require.NoError(t, err)
defer iter.Close()
keys, err := iter.Keys()
require.NoError(t, err)
require.Equal(t, []uint64{0, 1}, keys)
// test nil end in reverse
iter, err = m.IterateRaw(ctx, nil, nil, OrderDescending)
require.NoError(t, err)
defer iter.Close()
keys, err = iter.Keys()
require.NoError(t, err)
require.Equal(t, []uint64{2, 1, 0}, keys)
}
func Test_encodeKey(t *testing.T) {
prefix := "prefix"
number := []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}

6
go.mod
View File

@ -4,6 +4,7 @@ module github.com/cosmos/cosmos-sdk
require (
cosmossdk.io/api v0.2.6
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476
cosmossdk.io/core v0.4.0
cosmossdk.io/depinject v1.0.0-alpha.3
cosmossdk.io/errors v1.0.0-beta.7
@ -117,8 +118,10 @@ require (
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/gtank/ristretto255 v0.1.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
@ -138,8 +141,7 @@ require (
github.com/mitchellh/go-testing-interface v1.0.0 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mtibben/percent v0.2.1 // indirect
github.com/onsi/ginkgo v1.16.4 // indirect
github.com/onsi/gomega v1.20.0 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/pelletier/go-toml v1.9.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.5 // indirect
github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect

11
go.sum
View File

@ -48,6 +48,8 @@ cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgy
cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s=
cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU=
cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE=
cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas=
cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE=
cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw=
@ -311,7 +313,6 @@ github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh
github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos=
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0=
github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8=
@ -460,6 +461,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@ -471,6 +474,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
@ -669,12 +674,10 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc=
github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q=
github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec=
@ -1087,7 +1090,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@ -1183,7 +1185,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u
golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=

3
simapp/go.mod
View File

@ -26,6 +26,7 @@ require (
cloud.google.com/go/compute/metadata v0.2.1 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/storage v1.27.0 // indirect
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 // indirect
cosmossdk.io/errors v1.0.0-beta.7 // indirect
filippo.io/edwards25519 v1.0.0-rc.1 // indirect
github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect
@ -95,9 +96,11 @@ require (
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/gtank/ristretto255 v0.1.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-getter v1.6.2 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect

8
simapp/go.sum
View File

@ -50,6 +50,8 @@ cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU=
cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI=
cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 h1:q0AkHBZnYhsnnS3AmTUu1BOO+TH3oNOsXbG6oeExwvg=
cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58/go.mod h1:ztqtfnFSD3edvhNOAShzKod13nfKLM1sZj0uu0fo56w=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE=
cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas=
cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE=
cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw=
@ -463,6 +465,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@ -474,6 +478,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
@ -659,8 +665,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=

3
tests/go.mod
View File

@ -27,6 +27,7 @@ require (
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/storage v1.27.0 // indirect
cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 // indirect
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 // indirect
cosmossdk.io/core v0.4.0 // indirect
cosmossdk.io/errors v1.0.0-beta.7 // indirect
filippo.io/edwards25519 v1.0.0-rc.1 // indirect
@ -94,9 +95,11 @@ require (
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/gtank/ristretto255 v0.1.2 // indirect
github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
github.com/hashicorp/go-getter v1.6.2 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-multierror v1.1.1 // indirect
github.com/hashicorp/go-safetemp v1.0.0 // indirect
github.com/hashicorp/go-version v1.6.0 // indirect
github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect

8
tests/go.sum
View File

@ -50,6 +50,8 @@ cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU=
cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI=
cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 h1:q0AkHBZnYhsnnS3AmTUu1BOO+TH3oNOsXbG6oeExwvg=
cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58/go.mod h1:ztqtfnFSD3edvhNOAShzKod13nfKLM1sZj0uu0fo56w=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64=
cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE=
cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas=
cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE=
cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw=
@ -457,6 +459,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv
github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE=
github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
@ -468,6 +472,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ
github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo=
@ -649,8 +655,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi
github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs=
github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA=
github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=

223
types/query/collections_pagination.go Normal file
View File

@ -0,0 +1,223 @@
package query
import (
"context"
"fmt"
"cosmossdk.io/collections"
)
// Collection defines the minimum required API of a collection
// to work with pagination.
type Collection[K, V any] interface {
// IterateRaw allows to iterate over a raw set of byte keys.
IterateRaw(ctx context.Context, start, end []byte, order collections.Order) (collections.Iterator[K, V], error)
// KeyCodec exposes the KeyCodec of a collection, required to encode a collection key from and to bytes
// for pagination request and response.
KeyCodec() collections.KeyCodec[K]
}
// CollectionPaginate follows the same behaviour as Paginate but works on a Collection.
func CollectionPaginate[K, V any, C Collection[K, V]](
ctx context.Context,
coll C,
pageReq *PageRequest,
) ([]collections.KeyValue[K, V], *PageResponse, error) {
return CollectionFilteredPaginate[K, V](ctx, coll, pageReq, nil)
}
// CollectionFilteredPaginate works in the same way as FilteredPaginate but for collection types.
// A nil predicateFunc means no filtering is applied and results are collected as is.
func CollectionFilteredPaginate[K, V any, C Collection[K, V]](
ctx context.Context,
coll C,
pageReq *PageRequest,
predicateFunc func(key K, value V) (include bool),
) ([]collections.KeyValue[K, V], *PageResponse, error) {
if pageReq == nil {
pageReq = &PageRequest{}
}
offset := pageReq.Offset
key := pageReq.Key
limit := pageReq.Limit
countTotal := pageReq.CountTotal
reverse := pageReq.Reverse
if offset > 0 && key != nil {
return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
}
if limit == 0 {
limit = DefaultLimit
countTotal = true
}
if len(key) != 0 {
return collFilteredPaginateByKey(ctx, coll, key, reverse, limit, predicateFunc)
}
return collFilteredPaginateNoKey(ctx, coll, reverse, offset, limit, countTotal, predicateFunc)
}
// collFilteredPaginateNoKey applies the provided pagination on the collection when the starting key is not set.
// If predicateFunc is nil no filtering is applied.
func collFilteredPaginateNoKey[K, V any, C Collection[K, V]](
ctx context.Context,
coll C,
reverse bool,
offset uint64,
limit uint64,
countTotal bool,
predicateFunc func(K, V) bool,
) ([]collections.KeyValue[K, V], *PageResponse, error) {
iterator, err := getCollIter[K, V](ctx, coll, nil, reverse)
if err != nil {
return nil, nil, err
}
defer iterator.Close()
// we advance the iter equal to the provided offset
if !advanceIter(iterator, offset) {
return nil, nil, collections.ErrInvalidIterator
}
var (
count uint64
nextKey []byte
results []collections.KeyValue[K, V]
)
for ; iterator.Valid(); iterator.Next() {
switch {
// first case, we still haven't found all the results up to the limit
case count < limit:
kv, err := iterator.KeyValue()
if err != nil {
return nil, nil, err
}
// if no predicate function is specified then we just include the result
if predicateFunc == nil {
results = append(results, kv)
count++
// if predicate function is defined we check if the result matches the filtering criteria
} else if predicateFunc(kv.Key, kv.Value) {
results = append(results, kv)
count++
}
// second case, we found all the objects specified within the limit
case count == limit:
key, err := iterator.Key()
if err != nil {
return nil, nil, err
}
nextKey, err = encodeCollKey[K, V](coll, key)
if err != nil {
return nil, nil, err
}
// if count total was not specified, we return the next key only
if !countTotal {
return results, &PageResponse{
NextKey: nextKey,
}, nil
}
// otherwise we fallthrough the third case
fallthrough
// this is the case in which we found all the required results
// but we need to count how many possible results exist in total.
// so we keep increasing the count until the iterator is fully consumed.
case count > limit:
count++
}
}
return results, &PageResponse{
NextKey: nextKey,
Total: count + offset,
}, nil
}
func advanceIter[I interface {
Next()
Valid() bool
}](iter I, offset uint64,
) bool {
for i := uint64(0); i < offset; i++ {
if !iter.Valid() {
return false
}
iter.Next()
}
return true
}
// collFilteredPaginateByKey paginates a collection when a starting key
// is provided in the PageRequest. Predicate is applied only if not nil.
func collFilteredPaginateByKey[K, V any, C Collection[K, V]](
ctx context.Context,
coll C,
key []byte,
reverse bool,
limit uint64,
predicateFunc func(K, V) bool,
) ([]collections.KeyValue[K, V], *PageResponse, error) {
iterator, err := getCollIter[K, V](ctx, coll, key, reverse)
if err != nil {
return nil, nil, err
}
defer iterator.Close()
var (
count uint64
nextKey []byte
results []collections.KeyValue[K, V]
)
for ; iterator.Valid(); iterator.Next() {
// if we reached the specified limit
// then we get the next key, and we exit the iteration.
if count == limit {
concreteKey, err := iterator.Key()
if err != nil {
return nil, nil, err
}
nextKey, err = encodeCollKey[K, V](coll, concreteKey)
if err != nil {
return nil, nil, err
}
break
}
kv, err := iterator.KeyValue()
if err != nil {
return nil, nil, err
}
// if no predicate is specified then we just append the result
if predicateFunc == nil {
results = append(results, kv)
count++
// if predicate is applied we execute the predicate function
// and append only if predicateFunc yields true.
} else if predicateFunc(kv.Key, kv.Value) {
results = append(results, kv)
count++
}
}
return results, &PageResponse{
NextKey: nextKey,
}, nil
}
// todo maybe move to collections?
func encodeCollKey[K, V any](coll Collection[K, V], key K) ([]byte, error) {
buffer := make([]byte, coll.KeyCodec().Size(key))
_, err := coll.KeyCodec().Encode(buffer, key)
return buffer, err
}
func getCollIter[K, V any](ctx context.Context, coll Collection[K, V], start []byte, reverse bool) (collections.Iterator[K, V], error) {
if reverse {
return coll.IterateRaw(ctx, nil, start, collections.OrderDescending)
}
return coll.IterateRaw(ctx, start, nil, collections.OrderAscending)
}

210
types/query/collections_pagination_test.go Normal file
View File

@ -0,0 +1,210 @@
package query
import (
"context"
"testing"
"cosmossdk.io/collections"
"cosmossdk.io/core/store"
db "github.com/cosmos/cosmos-db"
"github.com/stretchr/testify/require"
)
func TestCollectionPagination(t *testing.T) {
sk, ctx := deps()
sb := collections.NewSchemaBuilder(sk)
m := collections.NewMap(sb, collections.NewPrefix(0), "_", collections.Uint64Key, collections.Uint64Value)
for i := uint64(0); i < 300; i++ {
require.NoError(t, m.Set(ctx, i, i))
}
createResults := func(from, to uint64) []collections.KeyValue[uint64, uint64] {
var res []collections.KeyValue[uint64, uint64]
if from <= to {
for i := from; i <= to; i++ {
res = append(res, collections.KeyValue[uint64, uint64]{
Key: i,
Value: i,
})
}
} else {
for i := from; i >= to; i-- {
res = append(res, collections.KeyValue[uint64, uint64]{
Key: i,
Value: i,
})
}
}
return res
}
encodeKey := func(key uint64) []byte {
b, err := encodeCollKey[uint64, uint64](m, key)
require.NoError(t, err)
return b
}
type test struct {
req *PageRequest
expResp *PageResponse
filter func(key uint64, value uint64) bool
expResults []collections.KeyValue[uint64, uint64]
wantErr error
}
tcs := map[string]test{
"nil pagination": {
req: nil,
expResp: &PageResponse{
NextKey: encodeKey(100),
Total: 300,
},
expResults: createResults(0, 99),
},
"with key and limit": {
req: &PageRequest{
Key: encodeKey(100),
Limit: 149,
},
expResp: &PageResponse{
NextKey: encodeKey(249),
},
expResults: createResults(100, 248),
},
"with reverse": {
req: &PageRequest{
Reverse: true,
},
expResp: &PageResponse{
NextKey: encodeKey(199),
Total: 300,
},
expResults: createResults(299, 200),
},
"with offset and count total": {
req: &PageRequest{
Offset: 50,
Limit: 100,
CountTotal: true,
},
expResp: &PageResponse{
NextKey: encodeKey(150),
Total: 300,
},
expResults: createResults(50, 149),
},
"filtered no key": {
req: &PageRequest{
Limit: 3,
},
expResp: &PageResponse{
NextKey: encodeKey(5),
},
filter: func(key uint64, value uint64) bool {
return key%2 == 0
},
expResults: []collections.KeyValue[uint64, uint64]{
{Key: 0, Value: 0},
{Key: 2, Value: 2},
{Key: 4, Value: 4},
},
},
"filtered with key": {
req: &PageRequest{
Key: encodeKey(2),
Limit: 3,
},
expResp: &PageResponse{
NextKey: encodeKey(7),
},
filter: func(key uint64, value uint64) bool {
return key%2 == 0
},
expResults: []collections.KeyValue[uint64, uint64]{
{Key: 2, Value: 2},
{Key: 4, Value: 4},
{Key: 6, Value: 6},
},
},
"error offset > total items": {
req: &PageRequest{Offset: 500},
wantErr: collections.ErrInvalidIterator,
},
}
for name, tc := range tcs {
tc := tc
t.Run(name, func(t *testing.T) {
gotResults, gotResponse, err := CollectionFilteredPaginate(ctx, m, tc.req, tc.filter)
if tc.wantErr != nil {
require.ErrorIs(t, err, tc.wantErr)
return
}
require.NoError(t, err)
require.Equal(t, tc.expResults, gotResults)
require.Equal(t, tc.expResp, gotResponse)
})
}
}
type testStore struct {
db db.DB
}
func (t testStore) OpenKVStore(ctx context.Context) store.KVStore {
return t
}
func (t testStore) Get(key []byte) []byte {
res, err := t.db.Get(key)
if err != nil {
panic(err)
}
return res
}
func (t testStore) Has(key []byte) bool {
res, err := t.db.Has(key)
if err != nil {
panic(err)
}
return res
}
func (t testStore) Set(key, value []byte) {
err := t.db.Set(key, value)
if err != nil {
panic(err)
}
}
func (t testStore) Delete(key []byte) {
err := t.db.Delete(key)
if err != nil {
panic(err)
}
}
func (t testStore) Iterator(start, end []byte) store.Iterator {
res, err := t.db.Iterator(start, end)
if err != nil {
panic(err)
}
return res
}
func (t testStore) ReverseIterator(start, end []byte) store.Iterator {
res, err := t.db.ReverseIterator(start, end)
if err != nil {
panic(err)
}
return res
}
var _ store.KVStore = testStore{}
func deps() (store.KVStoreService, context.Context) {
kv := db.NewMemDB()
return &testStore{kv}, context.Background()
}