diff --git a/CHANGELOG.md b/CHANGELOG.md index c74a4ae3c3..246503b621 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ ### Features +* (query) [#14468](https://github.com/cosmos/cosmos-sdk/pull/14468) Implement pagination for collections. * (x/bank) [#14045](https://github.com/cosmos/cosmos-sdk/pull/14045) Add CLI command `spendable-balances`, which also accepts the flag `--denom`. * (x/slashing, x/staking) [#14363](https://github.com/cosmos/cosmos-sdk/pull/14363) Add the infraction a validator commited type as an argument to a `SlashWithInfractionReason` keeper method. * (client) [#13867](https://github.com/cosmos/cosmos-sdk/pull/13867/) Wire AutoCLI commands with SimApp. diff --git a/collections/CHANGELOG.md b/collections/CHANGELOG.md index b69be22353..2deeab13e8 100644 --- a/collections/CHANGELOG.md +++ b/collections/CHANGELOG.md @@ -33,4 +33,5 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#14134](https://github.com/cosmos/cosmos-sdk/pull/14134) Initialise core (Prefix, KeyEncoder, ValueEncoder, Map). * [#14351](https://github.com/cosmos/cosmos-sdk/pull/14351) Add keyset -* [#14364](https://github.com/cosmos/cosmos-sdk/pull/14364) Add sequence \ No newline at end of file +* [#14364](https://github.com/cosmos/cosmos-sdk/pull/14364) Add sequence +* [#14468](https://github.com/cosmos/cosmos-sdk/pull/14468) Add Map.IterateRaw API. \ No newline at end of file diff --git a/collections/iter.go b/collections/iter.go index c99d3b9bc1..d16bfbd877 100644 --- a/collections/iter.go +++ b/collections/iter.go @@ -2,10 +2,9 @@ package collections import ( "context" + "cosmossdk.io/core/store" "errors" "fmt" - - "cosmossdk.io/core/store" ) // ErrInvalidIterator is returned when an Iterate call resulted in an invalid iterator. diff --git a/collections/iter_test.go b/collections/iter_test.go index a7aaeae48d..038efae764 100644 --- a/collections/iter_test.go +++ b/collections/iter_test.go @@ -9,8 +9,11 @@ import ( func TestIteratorBasic(t *testing.T) { sk, ctx := deps() + // safety check to ensure that iteration does not cross prefix boundaries + sk.OpenKVStore(ctx).Set([]byte{0, 0}, []byte("before prefix")) + sk.OpenKVStore(ctx).Set([]byte{2, 1}, []byte("after prefix")) schemaBuilder := NewSchemaBuilder(sk) - m := NewMap(schemaBuilder, NewPrefix("some super amazing prefix"), "m", StringKey, Uint64Value) + m := NewMap(schemaBuilder, NewPrefix(1), "m", StringKey, Uint64Value) _, err := schemaBuilder.Build() require.NoError(t, err) diff --git a/collections/map.go b/collections/map.go index 298baa4cdd..ee8428ded9 100644 --- a/collections/map.go +++ b/collections/map.go @@ -128,6 +128,49 @@ func (m Map[K, V]) Iterate(ctx context.Context, ranger Ranger[K]) (Iterator[K, V return iteratorFromRanger(ctx, m, ranger) } +// IterateRaw iterates over the collection. The iteration range is untyped, it uses raw +// bytes. The resulting Iterator is typed. +// A nil start iterates from the first key contained in the collection. +// A nil end iterates up to the last key contained in the collection. +// A nil start and a nil end iterates over every key contained in the collection. +// TODO(tip): simplify after https://github.com/cosmos/cosmos-sdk/pull/14310 is merged +func (m Map[K, V]) IterateRaw(ctx context.Context, start, end []byte, order Order) (Iterator[K, V], error) { + prefixedStart := append(m.prefix, start...) + var prefixedEnd []byte + if end == nil { + prefixedEnd = prefixEndBytes(m.prefix) + } else { + prefixedEnd = append(m.prefix, end...) + } + + s := m.sa(ctx) + var storeIter store.Iterator + switch order { + case OrderAscending: + storeIter = s.Iterator(prefixedStart, prefixedEnd) + case OrderDescending: + storeIter = s.ReverseIterator(prefixedStart, prefixedEnd) + default: + return Iterator[K, V]{}, errOrder + } + + if !storeIter.Valid() { + return Iterator[K, V]{}, ErrInvalidIterator + } + return Iterator[K, V]{ + kc: m.kc, + vc: m.vc, + iter: storeIter, + prefixLength: len(m.prefix), + }, nil +} + +// KeyCodec returns the Map's KeyCodec. +func (m Map[K, V]) KeyCodec() KeyCodec[K] { return m.kc } + +// ValueCodec returns the Map's ValueCodec. +func (m Map[K, V]) ValueCodec() ValueCodec[V] { return m.vc } + func encodeKeyWithPrefix[K any](prefix []byte, kc KeyCodec[K], key K) ([]byte, error) { prefixLen := len(prefix) // preallocate buffer diff --git a/collections/map_test.go b/collections/map_test.go index a01879cf38..a8746a4aa7 100644 --- a/collections/map_test.go +++ b/collections/map_test.go @@ -36,6 +36,41 @@ func TestMap(t *testing.T) { require.False(t, has) } +func TestMap_IterateRaw(t *testing.T) { + sk, ctx := deps() + // safety check to ensure prefix boundaries are not crossed + sk.OpenKVStore(ctx).Set([]byte{0x0, 0x0}, []byte("before prefix")) + sk.OpenKVStore(ctx).Set([]byte{0x2, 0x0}, []byte("after prefix")) + + sb := NewSchemaBuilder(sk) + + m := NewMap(sb, NewPrefix(1), "m", Uint64Key, Uint64Value) + require.NoError(t, m.Set(ctx, 0, 0)) + require.NoError(t, m.Set(ctx, 1, 1)) + require.NoError(t, m.Set(ctx, 2, 2)) + + // test non nil end in ascending order + twoBigEndian, err := encodeKeyWithPrefix(nil, Uint64Key, 2) + require.NoError(t, err) + iter, err := m.IterateRaw(ctx, nil, twoBigEndian, OrderAscending) + require.NoError(t, err) + defer iter.Close() + + keys, err := iter.Keys() + require.NoError(t, err) + + require.Equal(t, []uint64{0, 1}, keys) + + // test nil end in reverse + iter, err = m.IterateRaw(ctx, nil, nil, OrderDescending) + require.NoError(t, err) + defer iter.Close() + + keys, err = iter.Keys() + require.NoError(t, err) + require.Equal(t, []uint64{2, 1, 0}, keys) +} + func Test_encodeKey(t *testing.T) { prefix := "prefix" number := []byte{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0} diff --git a/go.mod b/go.mod index cf5d0ee5b7..15a08e4811 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ module github.com/cosmos/cosmos-sdk require ( cosmossdk.io/api v0.2.6 + cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 cosmossdk.io/core v0.4.0 cosmossdk.io/depinject v1.0.0-alpha.3 cosmossdk.io/errors v1.0.0-beta.7 @@ -117,8 +118,10 @@ require ( github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -138,8 +141,7 @@ require ( github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mtibben/percent v0.2.1 // indirect - github.com/onsi/ginkgo v1.16.4 // indirect - github.com/onsi/gomega v1.20.0 // indirect + github.com/nxadm/tail v1.4.8 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect diff --git a/go.sum b/go.sum index de1cb0e736..16a56ad72f 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgy cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU= cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE= cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas= cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE= cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw= @@ -311,7 +313,6 @@ github.com/go-playground/validator/v10 v10.10.0 h1:I7mrTYv78z8k8VXa/qJlOlEXn/nBh github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXSGrTK4nAUsbPlLADvpJkos= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= @@ -460,6 +461,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -471,6 +474,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -669,12 +674,10 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= -github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= -github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 h1:rc3tiVYb5z54aKaDfakKn0dDjIyPpTtszkjuMzyt7ec= @@ -1087,7 +1090,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1183,7 +1185,6 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= diff --git a/simapp/go.mod b/simapp/go.mod index 4e5f0b8028..446b61aa18 100644 --- a/simapp/go.mod +++ b/simapp/go.mod @@ -26,6 +26,7 @@ require ( cloud.google.com/go/compute/metadata v0.2.1 // indirect cloud.google.com/go/iam v0.8.0 // indirect cloud.google.com/go/storage v1.27.0 // indirect + cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 // indirect cosmossdk.io/errors v1.0.0-beta.7 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -95,9 +96,11 @@ require ( github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.6.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect diff --git a/simapp/go.sum b/simapp/go.sum index d2e38a6f89..149c50796f 100644 --- a/simapp/go.sum +++ b/simapp/go.sum @@ -50,6 +50,8 @@ cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU= cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI= cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 h1:q0AkHBZnYhsnnS3AmTUu1BOO+TH3oNOsXbG6oeExwvg= cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58/go.mod h1:ztqtfnFSD3edvhNOAShzKod13nfKLM1sZj0uu0fo56w= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE= cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas= cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE= cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw= @@ -463,6 +465,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -474,6 +478,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -659,8 +665,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= diff --git a/tests/go.mod b/tests/go.mod index f88273d458..91069e19e2 100644 --- a/tests/go.mod +++ b/tests/go.mod @@ -27,6 +27,7 @@ require ( cloud.google.com/go/iam v0.8.0 // indirect cloud.google.com/go/storage v1.27.0 // indirect cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 // indirect + cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 // indirect cosmossdk.io/core v0.4.0 // indirect cosmossdk.io/errors v1.0.0-beta.7 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect @@ -94,9 +95,11 @@ require ( github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-getter v1.6.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect diff --git a/tests/go.sum b/tests/go.sum index 5dc226a9e6..f0841ffd3f 100644 --- a/tests/go.sum +++ b/tests/go.sum @@ -50,6 +50,8 @@ cosmossdk.io/api v0.2.6 h1:AoNwaLLapcLsphhMK6+o0kZl+D6MMUaHVqSdwinASGU= cosmossdk.io/api v0.2.6/go.mod h1:u/d+GAxil0nWpl1XnQL8nkziQDIWuBDhv8VnDm/s6dI= cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58 h1:q0AkHBZnYhsnnS3AmTUu1BOO+TH3oNOsXbG6oeExwvg= cosmossdk.io/client/v2 v2.0.0-20230104083136-11f46a0bae58/go.mod h1:ztqtfnFSD3edvhNOAShzKod13nfKLM1sZj0uu0fo56w= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476 h1:dszme7EMNp/qxjgZKjhY4lm5SN+g1Tz95UlX/qZUK64= +cosmossdk.io/collections v0.0.0-20230106101515-aeac21494476/go.mod h1:q4avveOJ2Cbgab/Hm/yXDHFPM7NW9InEVysd8TzNPCE= cosmossdk.io/core v0.4.0 h1:QPYg2v21OGr7yjJC5HEWiuhfFJxzdqbDDv1PZzH7cas= cosmossdk.io/core v0.4.0/go.mod h1:Zqd1GB+krTF3bzhTnPNwzy3NTtXu+MKLX/9sPQXTIDE= cosmossdk.io/depinject v1.0.0-alpha.3 h1:6evFIgj//Y3w09bqOUOzEpFj5tsxBqdc5CfkO7z+zfw= @@ -457,6 +459,8 @@ github.com/gtank/ristretto255 v0.1.2/go.mod h1:Ph5OpO6c7xKUGROZfWVLiJf9icMDwUeIv github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -468,6 +472,8 @@ github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJ github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -649,8 +655,8 @@ github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= diff --git a/types/query/collections_pagination.go b/types/query/collections_pagination.go new file mode 100644 index 0000000000..c39b83c590 --- /dev/null +++ b/types/query/collections_pagination.go @@ -0,0 +1,223 @@ +package query + +import ( + "context" + "fmt" + + "cosmossdk.io/collections" +) + +// Collection defines the minimum required API of a collection +// to work with pagination. +type Collection[K, V any] interface { + // IterateRaw allows to iterate over a raw set of byte keys. + IterateRaw(ctx context.Context, start, end []byte, order collections.Order) (collections.Iterator[K, V], error) + // KeyCodec exposes the KeyCodec of a collection, required to encode a collection key from and to bytes + // for pagination request and response. + KeyCodec() collections.KeyCodec[K] +} + +// CollectionPaginate follows the same behaviour as Paginate but works on a Collection. +func CollectionPaginate[K, V any, C Collection[K, V]]( + ctx context.Context, + coll C, + pageReq *PageRequest, +) ([]collections.KeyValue[K, V], *PageResponse, error) { + return CollectionFilteredPaginate[K, V](ctx, coll, pageReq, nil) +} + +// CollectionFilteredPaginate works in the same way as FilteredPaginate but for collection types. +// A nil predicateFunc means no filtering is applied and results are collected as is. +func CollectionFilteredPaginate[K, V any, C Collection[K, V]]( + ctx context.Context, + coll C, + pageReq *PageRequest, + predicateFunc func(key K, value V) (include bool), +) ([]collections.KeyValue[K, V], *PageResponse, error) { + if pageReq == nil { + pageReq = &PageRequest{} + } + + offset := pageReq.Offset + key := pageReq.Key + limit := pageReq.Limit + countTotal := pageReq.CountTotal + reverse := pageReq.Reverse + + if offset > 0 && key != nil { + return nil, nil, fmt.Errorf("invalid request, either offset or key is expected, got both") + } + + if limit == 0 { + limit = DefaultLimit + countTotal = true + } + + if len(key) != 0 { + return collFilteredPaginateByKey(ctx, coll, key, reverse, limit, predicateFunc) + } + + return collFilteredPaginateNoKey(ctx, coll, reverse, offset, limit, countTotal, predicateFunc) +} + +// collFilteredPaginateNoKey applies the provided pagination on the collection when the starting key is not set. +// If predicateFunc is nil no filtering is applied. +func collFilteredPaginateNoKey[K, V any, C Collection[K, V]]( + ctx context.Context, + coll C, + reverse bool, + offset uint64, + limit uint64, + countTotal bool, + predicateFunc func(K, V) bool, +) ([]collections.KeyValue[K, V], *PageResponse, error) { + iterator, err := getCollIter[K, V](ctx, coll, nil, reverse) + if err != nil { + return nil, nil, err + } + defer iterator.Close() + // we advance the iter equal to the provided offset + if !advanceIter(iterator, offset) { + return nil, nil, collections.ErrInvalidIterator + } + + var ( + count uint64 + nextKey []byte + results []collections.KeyValue[K, V] + ) + + for ; iterator.Valid(); iterator.Next() { + switch { + // first case, we still haven't found all the results up to the limit + case count < limit: + kv, err := iterator.KeyValue() + if err != nil { + return nil, nil, err + } + // if no predicate function is specified then we just include the result + if predicateFunc == nil { + results = append(results, kv) + count++ + // if predicate function is defined we check if the result matches the filtering criteria + } else if predicateFunc(kv.Key, kv.Value) { + results = append(results, kv) + count++ + } + // second case, we found all the objects specified within the limit + case count == limit: + key, err := iterator.Key() + if err != nil { + return nil, nil, err + } + nextKey, err = encodeCollKey[K, V](coll, key) + if err != nil { + return nil, nil, err + } + // if count total was not specified, we return the next key only + if !countTotal { + return results, &PageResponse{ + NextKey: nextKey, + }, nil + } + // otherwise we fallthrough the third case + fallthrough + // this is the case in which we found all the required results + // but we need to count how many possible results exist in total. + // so we keep increasing the count until the iterator is fully consumed. + case count > limit: + count++ + } + } + return results, &PageResponse{ + NextKey: nextKey, + Total: count + offset, + }, nil +} + +func advanceIter[I interface { + Next() + Valid() bool +}](iter I, offset uint64, +) bool { + for i := uint64(0); i < offset; i++ { + if !iter.Valid() { + return false + } + iter.Next() + } + return true +} + +// collFilteredPaginateByKey paginates a collection when a starting key +// is provided in the PageRequest. Predicate is applied only if not nil. +func collFilteredPaginateByKey[K, V any, C Collection[K, V]]( + ctx context.Context, + coll C, + key []byte, + reverse bool, + limit uint64, + predicateFunc func(K, V) bool, +) ([]collections.KeyValue[K, V], *PageResponse, error) { + iterator, err := getCollIter[K, V](ctx, coll, key, reverse) + if err != nil { + return nil, nil, err + } + defer iterator.Close() + + var ( + count uint64 + nextKey []byte + results []collections.KeyValue[K, V] + ) + + for ; iterator.Valid(); iterator.Next() { + // if we reached the specified limit + // then we get the next key, and we exit the iteration. + if count == limit { + concreteKey, err := iterator.Key() + if err != nil { + return nil, nil, err + } + + nextKey, err = encodeCollKey[K, V](coll, concreteKey) + if err != nil { + return nil, nil, err + } + break + } + + kv, err := iterator.KeyValue() + if err != nil { + return nil, nil, err + } + // if no predicate is specified then we just append the result + if predicateFunc == nil { + results = append(results, kv) + count++ + // if predicate is applied we execute the predicate function + // and append only if predicateFunc yields true. + } else if predicateFunc(kv.Key, kv.Value) { + results = append(results, kv) + count++ + } + } + + return results, &PageResponse{ + NextKey: nextKey, + }, nil +} + +// todo maybe move to collections? +func encodeCollKey[K, V any](coll Collection[K, V], key K) ([]byte, error) { + buffer := make([]byte, coll.KeyCodec().Size(key)) + _, err := coll.KeyCodec().Encode(buffer, key) + return buffer, err +} + +func getCollIter[K, V any](ctx context.Context, coll Collection[K, V], start []byte, reverse bool) (collections.Iterator[K, V], error) { + if reverse { + return coll.IterateRaw(ctx, nil, start, collections.OrderDescending) + } + return coll.IterateRaw(ctx, start, nil, collections.OrderAscending) +} diff --git a/types/query/collections_pagination_test.go b/types/query/collections_pagination_test.go new file mode 100644 index 0000000000..ab0661f57b --- /dev/null +++ b/types/query/collections_pagination_test.go @@ -0,0 +1,210 @@ +package query + +import ( + "context" + "testing" + + "cosmossdk.io/collections" + "cosmossdk.io/core/store" + db "github.com/cosmos/cosmos-db" + "github.com/stretchr/testify/require" +) + +func TestCollectionPagination(t *testing.T) { + sk, ctx := deps() + sb := collections.NewSchemaBuilder(sk) + m := collections.NewMap(sb, collections.NewPrefix(0), "_", collections.Uint64Key, collections.Uint64Value) + + for i := uint64(0); i < 300; i++ { + require.NoError(t, m.Set(ctx, i, i)) + } + + createResults := func(from, to uint64) []collections.KeyValue[uint64, uint64] { + var res []collections.KeyValue[uint64, uint64] + if from <= to { + for i := from; i <= to; i++ { + res = append(res, collections.KeyValue[uint64, uint64]{ + Key: i, + Value: i, + }) + } + } else { + for i := from; i >= to; i-- { + res = append(res, collections.KeyValue[uint64, uint64]{ + Key: i, + Value: i, + }) + } + } + return res + } + + encodeKey := func(key uint64) []byte { + b, err := encodeCollKey[uint64, uint64](m, key) + require.NoError(t, err) + return b + } + + type test struct { + req *PageRequest + expResp *PageResponse + filter func(key uint64, value uint64) bool + expResults []collections.KeyValue[uint64, uint64] + wantErr error + } + + tcs := map[string]test{ + "nil pagination": { + req: nil, + expResp: &PageResponse{ + NextKey: encodeKey(100), + Total: 300, + }, + expResults: createResults(0, 99), + }, + "with key and limit": { + req: &PageRequest{ + Key: encodeKey(100), + Limit: 149, + }, + expResp: &PageResponse{ + NextKey: encodeKey(249), + }, + expResults: createResults(100, 248), + }, + "with reverse": { + req: &PageRequest{ + Reverse: true, + }, + expResp: &PageResponse{ + NextKey: encodeKey(199), + Total: 300, + }, + expResults: createResults(299, 200), + }, + "with offset and count total": { + req: &PageRequest{ + Offset: 50, + Limit: 100, + CountTotal: true, + }, + expResp: &PageResponse{ + NextKey: encodeKey(150), + Total: 300, + }, + expResults: createResults(50, 149), + }, + "filtered no key": { + req: &PageRequest{ + Limit: 3, + }, + expResp: &PageResponse{ + NextKey: encodeKey(5), + }, + filter: func(key uint64, value uint64) bool { + return key%2 == 0 + }, + expResults: []collections.KeyValue[uint64, uint64]{ + {Key: 0, Value: 0}, + {Key: 2, Value: 2}, + {Key: 4, Value: 4}, + }, + }, + "filtered with key": { + req: &PageRequest{ + Key: encodeKey(2), + Limit: 3, + }, + expResp: &PageResponse{ + NextKey: encodeKey(7), + }, + filter: func(key uint64, value uint64) bool { + return key%2 == 0 + }, + expResults: []collections.KeyValue[uint64, uint64]{ + {Key: 2, Value: 2}, + {Key: 4, Value: 4}, + {Key: 6, Value: 6}, + }, + }, + "error offset > total items": { + req: &PageRequest{Offset: 500}, + wantErr: collections.ErrInvalidIterator, + }, + } + + for name, tc := range tcs { + tc := tc + t.Run(name, func(t *testing.T) { + gotResults, gotResponse, err := CollectionFilteredPaginate(ctx, m, tc.req, tc.filter) + if tc.wantErr != nil { + require.ErrorIs(t, err, tc.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, tc.expResults, gotResults) + require.Equal(t, tc.expResp, gotResponse) + }) + } +} + +type testStore struct { + db db.DB +} + +func (t testStore) OpenKVStore(ctx context.Context) store.KVStore { + return t +} + +func (t testStore) Get(key []byte) []byte { + res, err := t.db.Get(key) + if err != nil { + panic(err) + } + return res +} + +func (t testStore) Has(key []byte) bool { + res, err := t.db.Has(key) + if err != nil { + panic(err) + } + return res +} + +func (t testStore) Set(key, value []byte) { + err := t.db.Set(key, value) + if err != nil { + panic(err) + } +} + +func (t testStore) Delete(key []byte) { + err := t.db.Delete(key) + if err != nil { + panic(err) + } +} + +func (t testStore) Iterator(start, end []byte) store.Iterator { + res, err := t.db.Iterator(start, end) + if err != nil { + panic(err) + } + return res +} + +func (t testStore) ReverseIterator(start, end []byte) store.Iterator { + res, err := t.db.ReverseIterator(start, end) + if err != nil { + panic(err) + } + return res +} + +var _ store.KVStore = testStore{} + +func deps() (store.KVStoreService, context.Context) { + kv := db.NewMemDB() + return &testStore{kv}, context.Background() +}