From df43322ef3a420cbf507628d3217bab5b6c61dc7 Mon Sep 17 00:00:00 2001 From: Prathamesh Musale Date: Thu, 5 Sep 2024 07:33:48 +0000 Subject: [PATCH] Add pagination for query to get records (#58) Part of [Create a public laconicd testnet](https://www.notion.so/Create-a-public-laconicd-testnet-896a11bdd8094eff8f1b49c0be0ca3b8) Handles https://git.vdb.to/cerc-io/laconic-console/issues/59 Co-authored-by: IshaVenikar Reviewed-on: https://git.vdb.to/cerc-io/laconicd/pulls/58 Co-authored-by: Prathamesh Musale Co-committed-by: Prathamesh Musale --- gql/cerc-io/laconicd/schema.graphql | 10 +- gql/generated.go | 152 +++++++++++++--------------- gql/resolver.go | 28 ++++- scripts/init.sh | 2 +- x/onboarding/keeper/keeper.go | 2 +- x/registry/keeper/genesis.go | 2 +- x/registry/keeper/keeper.go | 124 +++++++++++++++++++---- x/registry/keeper/query_server.go | 8 +- 8 files changed, 216 insertions(+), 112 deletions(-) diff --git a/gql/cerc-io/laconicd/schema.graphql b/gql/cerc-io/laconicd/schema.graphql index c3fac8f6..288ebe37 100644 --- a/gql/cerc-io/laconicd/schema.graphql +++ b/gql/cerc-io/laconicd/schema.graphql @@ -227,7 +227,7 @@ type Query { getBondsByIds(ids: [String!]): [Bond] # Query bonds. - queryBonds(attributes: [KeyValueInput!]): [Bond] + queryBonds: [Bond] # Query bonds by owner. queryBondsByOwner(ownerAddresses: [String!]): [OwnerBonds] @@ -246,6 +246,12 @@ type Query { # Whether to query all records, not just named ones (false by default). all: Boolean + + # Pagination limit + limit: Int + + # Pagination offset + offset: Int ): [Record] # @@ -253,7 +259,7 @@ type Query { # # Get authorities list. - getAuthorities(owner: String): [Authority] + getAuthorities(owner: String): [Authority]! # Lookup authority information. lookupAuthorities(names: [String!]): [AuthorityRecord]! diff --git a/gql/generated.go b/gql/generated.go index a243f6df..a21c18a3 100644 --- a/gql/generated.go +++ b/gql/generated.go @@ -183,9 +183,9 @@ type ComplexityRoot struct { GetStatus func(childComplexity int) int LookupAuthorities func(childComplexity int, names []string) int LookupNames func(childComplexity int, names []string) int - QueryBonds func(childComplexity int, attributes []*KeyValueInput) int + QueryBonds func(childComplexity int) int QueryBondsByOwner func(childComplexity int, ownerAddresses []string) int - QueryRecords func(childComplexity int, attributes []*KeyValueInput, all *bool) int + QueryRecords func(childComplexity int, attributes []*KeyValueInput, all *bool, limit *int, offset *int) int ResolveNames func(childComplexity int, names []string) int } @@ -233,10 +233,10 @@ type QueryResolver interface { GetStatus(ctx context.Context) (*Status, error) GetAccounts(ctx context.Context, addresses []string) ([]*Account, error) GetBondsByIds(ctx context.Context, ids []string) ([]*Bond, error) - QueryBonds(ctx context.Context, attributes []*KeyValueInput) ([]*Bond, error) + QueryBonds(ctx context.Context) ([]*Bond, error) QueryBondsByOwner(ctx context.Context, ownerAddresses []string) ([]*OwnerBonds, error) GetRecordsByIds(ctx context.Context, ids []string) ([]*Record, error) - QueryRecords(ctx context.Context, attributes []*KeyValueInput, all *bool) ([]*Record, error) + QueryRecords(ctx context.Context, attributes []*KeyValueInput, all *bool, limit *int, offset *int) ([]*Record, error) GetAuthorities(ctx context.Context, owner *string) ([]*Authority, error) LookupAuthorities(ctx context.Context, names []string) ([]*AuthorityRecord, error) LookupNames(ctx context.Context, names []string) ([]*NameRecord, error) @@ -844,12 +844,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in break } - args, err := ec.field_Query_queryBonds_args(context.TODO(), rawArgs) - if err != nil { - return 0, false - } - - return e.complexity.Query.QueryBonds(childComplexity, args["attributes"].([]*KeyValueInput)), true + return e.complexity.Query.QueryBonds(childComplexity), true case "Query.queryBondsByOwner": if e.complexity.Query.QueryBondsByOwner == nil { @@ -873,7 +868,7 @@ func (e *executableSchema) Complexity(typeName, field string, childComplexity in return 0, false } - return e.complexity.Query.QueryRecords(childComplexity, args["attributes"].([]*KeyValueInput), args["all"].(*bool)), true + return e.complexity.Query.QueryRecords(childComplexity, args["attributes"].([]*KeyValueInput), args["all"].(*bool), args["limit"].(*int), args["offset"].(*int)), true case "Query.resolveNames": if e.complexity.Query.ResolveNames == nil { @@ -1294,21 +1289,6 @@ func (ec *executionContext) field_Query_queryBondsByOwner_args(ctx context.Conte return args, nil } -func (ec *executionContext) field_Query_queryBonds_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { - var err error - args := map[string]interface{}{} - var arg0 []*KeyValueInput - if tmp, ok := rawArgs["attributes"]; ok { - ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("attributes")) - arg0, err = ec.unmarshalOKeyValueInput2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐKeyValueInputᚄ(ctx, tmp) - if err != nil { - return nil, err - } - } - args["attributes"] = arg0 - return args, nil -} - func (ec *executionContext) field_Query_queryRecords_args(ctx context.Context, rawArgs map[string]interface{}) (map[string]interface{}, error) { var err error args := map[string]interface{}{} @@ -1330,6 +1310,24 @@ func (ec *executionContext) field_Query_queryRecords_args(ctx context.Context, r } } args["all"] = arg1 + var arg2 *int + if tmp, ok := rawArgs["limit"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("limit")) + arg2, err = ec.unmarshalOInt2ᚖint(ctx, tmp) + if err != nil { + return nil, err + } + } + args["limit"] = arg2 + var arg3 *int + if tmp, ok := rawArgs["offset"]; ok { + ctx := graphql.WithPathContext(ctx, graphql.NewPathWithField("offset")) + arg3, err = ec.unmarshalOInt2ᚖint(ctx, tmp) + if err != nil { + return nil, err + } + } + args["offset"] = arg3 return args, nil } @@ -4578,7 +4576,7 @@ func (ec *executionContext) _Query_queryBonds(ctx context.Context, field graphql }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().QueryBonds(rctx, fc.Args["attributes"].([]*KeyValueInput)) + return ec.resolvers.Query().QueryBonds(rctx) }) if err != nil { ec.Error(ctx, err) @@ -4610,17 +4608,6 @@ func (ec *executionContext) fieldContext_Query_queryBonds(ctx context.Context, f return nil, fmt.Errorf("no field named %q was found under type Bond", field.Name) }, } - defer func() { - if r := recover(); r != nil { - err = ec.Recover(ctx, r) - ec.Error(ctx, err) - } - }() - ctx = graphql.WithFieldContext(ctx, fc) - if fc.Args, err = ec.field_Query_queryBonds_args(ctx, field.ArgumentMap(ec.Variables)); err != nil { - ec.Error(ctx, err) - return - } return fc, nil } @@ -4766,7 +4753,7 @@ func (ec *executionContext) _Query_queryRecords(ctx context.Context, field graph }() resTmp, err := ec.ResolverMiddleware(ctx, func(rctx context.Context) (interface{}, error) { ctx = rctx // use context from middleware stack in children - return ec.resolvers.Query().QueryRecords(rctx, fc.Args["attributes"].([]*KeyValueInput), fc.Args["all"].(*bool)) + return ec.resolvers.Query().QueryRecords(rctx, fc.Args["attributes"].([]*KeyValueInput), fc.Args["all"].(*bool), fc.Args["limit"].(*int), fc.Args["offset"].(*int)) }) if err != nil { ec.Error(ctx, err) @@ -4843,11 +4830,14 @@ func (ec *executionContext) _Query_getAuthorities(ctx context.Context, field gra return graphql.Null } if resTmp == nil { + if !graphql.HasFieldError(ctx, fc) { + ec.Errorf(ctx, "must not be null") + } return graphql.Null } res := resTmp.([]*Authority) fc.Result = res - return ec.marshalOAuthority2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx, field.Selections, res) + return ec.marshalNAuthority2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx, field.Selections, res) } func (ec *executionContext) fieldContext_Query_getAuthorities(ctx context.Context, field graphql.CollectedField) (fc *graphql.FieldContext, err error) { @@ -9581,6 +9571,9 @@ func (ec *executionContext) _Query(ctx context.Context, sel ast.SelectionSet) gr } }() res = ec._Query_getAuthorities(ctx, field) + if res == graphql.Null { + atomic.AddUint32(&invalids, 1) + } return res } @@ -10396,6 +10389,44 @@ func (ec *executionContext) marshalNAttribute2ᚖgitᚗvdbᚗtoᚋcercᚑioᚋla return ec._Attribute(ctx, sel, v) } +func (ec *executionContext) marshalNAuthority2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx context.Context, sel ast.SelectionSet, v []*Authority) graphql.Marshaler { + ret := make(graphql.Array, len(v)) + var wg sync.WaitGroup + isLen1 := len(v) == 1 + if !isLen1 { + wg.Add(len(v)) + } + for i := range v { + i := i + fc := &graphql.FieldContext{ + Index: &i, + Result: &v[i], + } + ctx := graphql.WithFieldContext(ctx, fc) + f := func(i int) { + defer func() { + if r := recover(); r != nil { + ec.Error(ctx, ec.Recover(ctx, r)) + ret = nil + } + }() + if !isLen1 { + defer wg.Done() + } + ret[i] = ec.marshalOAuthority2ᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx, sel, v[i]) + } + if isLen1 { + f(i) + } else { + go f(i) + } + + } + wg.Wait() + + return ret +} + func (ec *executionContext) marshalNAuthorityRecord2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthorityRecord(ctx context.Context, sel ast.SelectionSet, v []*AuthorityRecord) graphql.Marshaler { ret := make(graphql.Array, len(v)) var wg sync.WaitGroup @@ -11231,47 +11262,6 @@ func (ec *executionContext) marshalOAuctionBid2ᚖgitᚗvdbᚗtoᚋcercᚑioᚋl return ec._AuctionBid(ctx, sel, v) } -func (ec *executionContext) marshalOAuthority2ᚕᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx context.Context, sel ast.SelectionSet, v []*Authority) graphql.Marshaler { - if v == nil { - return graphql.Null - } - ret := make(graphql.Array, len(v)) - var wg sync.WaitGroup - isLen1 := len(v) == 1 - if !isLen1 { - wg.Add(len(v)) - } - for i := range v { - i := i - fc := &graphql.FieldContext{ - Index: &i, - Result: &v[i], - } - ctx := graphql.WithFieldContext(ctx, fc) - f := func(i int) { - defer func() { - if r := recover(); r != nil { - ec.Error(ctx, ec.Recover(ctx, r)) - ret = nil - } - }() - if !isLen1 { - defer wg.Done() - } - ret[i] = ec.marshalOAuthority2ᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx, sel, v[i]) - } - if isLen1 { - f(i) - } else { - go f(i) - } - - } - wg.Wait() - - return ret -} - func (ec *executionContext) marshalOAuthority2ᚖgitᚗvdbᚗtoᚋcercᚑioᚋlaconicdᚋgqlᚐAuthority(ctx context.Context, sel ast.SelectionSet, v *Authority) graphql.Marshaler { if v == nil { return graphql.Null diff --git a/gql/resolver.go b/gql/resolver.go index 5a7cb106..f5aaf055 100644 --- a/gql/resolver.go +++ b/gql/resolver.go @@ -8,6 +8,7 @@ import ( "github.com/cosmos/cosmos-sdk/client" types "github.com/cosmos/cosmos-sdk/types" + "github.com/cosmos/cosmos-sdk/types/query" authtypes "github.com/cosmos/cosmos-sdk/x/auth/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" @@ -23,6 +24,9 @@ const DefaultLogNumLines = 50 // MaxLogNumLines is the max number of log lines that can be tailed. const MaxLogNumLines = 1000 +// Whether to use default page limit when pagination args are not passed. +const UseDefaultPagination = false + type Resolver struct { ctx client.Context logFile string @@ -136,14 +140,34 @@ func (q queryResolver) LookupNames(ctx context.Context, names []string) ([]*Name return gqlResponse, nil } -func (q queryResolver) QueryRecords(ctx context.Context, attributes []*KeyValueInput, all *bool) ([]*Record, error) { +func (q queryResolver) QueryRecords(ctx context.Context, attributes []*KeyValueInput, all *bool, limit *int, offset *int) ([]*Record, error) { nsQueryClient := registrytypes.NewQueryClient(q.ctx) + var pagination *query.PageRequest + + // Use defaults only if limit and offset not provided + // and UseDefaultPagination is true + if limit == nil && offset == nil { + if UseDefaultPagination { + pagination = &query.PageRequest{} + } + } else { + pagination = &query.PageRequest{} + + if limit != nil { + pagination.Limit = uint64(*limit) + } + if offset != nil { + pagination.Offset = uint64(*offset) + } + } + res, err := nsQueryClient.Records( context.Background(), ®istrytypes.QueryRecordsRequest{ Attributes: toRPCAttributes(attributes), All: (all != nil && *all), + Pagination: pagination, }, ) if err != nil { @@ -296,7 +320,7 @@ func (q *queryResolver) GetBond(ctx context.Context, id string) (*Bond, error) { return getGQLBond(bondResp.GetBond()) } -func (q queryResolver) QueryBonds(ctx context.Context, attributes []*KeyValueInput) ([]*Bond, error) { +func (q queryResolver) QueryBonds(ctx context.Context) ([]*Bond, error) { bondQueryClient := bondtypes.NewQueryClient(q.ctx) bonds, err := bondQueryClient.Bonds(context.Background(), &bondtypes.QueryBondsRequest{}) if err != nil { diff --git a/scripts/init.sh b/scripts/init.sh index d612eb4f..e41ea227 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -62,7 +62,7 @@ if [ "$1" == "clean" ] || [ ! -d "$HOME/.laconicd/data/blockstore.db" ]; then fi if [[ "$ONBOARDING_ENABLED" == "true" ]]; then - echo "Enabling validator onboarding." + echo "Enabling onboarding." update_genesis '.app_state["onboarding"]["params"]["onboarding_enabled"]=true' fi diff --git a/x/onboarding/keeper/keeper.go b/x/onboarding/keeper/keeper.go index d22b7af7..f53e0ca1 100644 --- a/x/onboarding/keeper/keeper.go +++ b/x/onboarding/keeper/keeper.go @@ -80,7 +80,7 @@ func (k Keeper) OnboardParticipant( } if !params.OnboardingEnabled { - return nil, errorsmod.Wrap(sdkerrors.ErrInvalidRequest, "Validator onboarding is disabled") + return nil, errorsmod.Wrap(sdkerrors.ErrInvalidRequest, "Onboarding is disabled") } message, err := json.Marshal(msg.EthPayload) diff --git a/x/registry/keeper/genesis.go b/x/registry/keeper/genesis.go index 8b6c2e74..36ec275d 100644 --- a/x/registry/keeper/genesis.go +++ b/x/registry/keeper/genesis.go @@ -62,7 +62,7 @@ func (k *Keeper) ExportGenesis(ctx sdk.Context) (*registry.GenesisState, error) return nil, err } - records, err := k.ListRecords(ctx) + records, _, err := k.PaginatedListRecords(ctx, nil) if err != nil { return nil, err } diff --git a/x/registry/keeper/keeper.go b/x/registry/keeper/keeper.go index b7e85bf3..89540541 100644 --- a/x/registry/keeper/keeper.go +++ b/x/registry/keeper/keeper.go @@ -16,6 +16,7 @@ import ( "github.com/cosmos/cosmos-sdk/codec/legacy" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/types/query" auth "github.com/cosmos/cosmos-sdk/x/auth/keeper" bank "github.com/cosmos/cosmos-sdk/x/bank/keeper" "github.com/gibson042/canonicaljson-go" @@ -204,23 +205,38 @@ func (k Keeper) HasRecord(ctx sdk.Context, id string) (bool, error) { return has, nil } -// ListRecords - get all records. -func (k Keeper) ListRecords(ctx sdk.Context) ([]registrytypes.Record, error) { +// PaginatedListRecords - get all records with optional pagination. +func (k Keeper) PaginatedListRecords(ctx sdk.Context, pagination *query.PageRequest) ([]registrytypes.Record, *query.PageResponse, error) { var records []registrytypes.Record + var pageResp *query.PageResponse - err := k.Records.Walk(ctx, nil, func(key string, value registrytypes.Record) (bool, error) { - if err := k.populateRecordNames(ctx, &value); err != nil { - return true, err + if pagination == nil { + err := k.Records.Walk(ctx, nil, func(key string, value registrytypes.Record) (bool, error) { + if err := k.populateRecordNames(ctx, &value); err != nil { + return true, err + } + records = append(records, value) + + return false, nil + }) + if err != nil { + return nil, nil, err } - records = append(records, value) + } else { + var err error + records, pageResp, err = query.CollectionPaginate(ctx, k.Records, pagination, func(key string, value registrytypes.Record) (registrytypes.Record, error) { + if err := k.populateRecordNames(ctx, &value); err != nil { + return registrytypes.Record{}, err + } - return false, nil - }) - if err != nil { - return nil, err + return value, nil + }) + if err != nil { + return nil, nil, err + } } - return records, nil + return records, pageResp, nil } // GetRecordById - gets a record from the store. @@ -261,36 +277,47 @@ func (k Keeper) GetRecordsByBondId(ctx sdk.Context, bondId string) ([]registryty return records, nil } -// RecordsFromAttributes gets a list of records whose attributes match all provided values -func (k Keeper) RecordsFromAttributes( +// PaginatedRecordsFromAttributes gets a list of records whose attributes match all provided values +// with optional pagination. +func (k Keeper) PaginatedRecordsFromAttributes( ctx sdk.Context, attributes []*registrytypes.QueryRecordsRequest_KeyValueInput, all bool, -) ([]registrytypes.Record, error) { - resultRecordIds := []string{} + pagination *query.PageRequest, +) ([]registrytypes.Record, *query.PageResponse, error) { + var resultRecordIds []string + var pageResp *query.PageResponse + + filteredRecordIds := []string{} for i, attr := range attributes { suffix, err := QueryValueToJSON(attr.Value) if err != nil { - return nil, err + return nil, nil, err } mapKey := collections.Join(attr.Key, string(suffix)) recordIds, err := k.getAttributeMapping(ctx, mapKey) if err != nil { - return nil, err + return nil, nil, err } if i == 0 { - resultRecordIds = recordIds + filteredRecordIds = recordIds } else { - resultRecordIds = getIntersection(recordIds, resultRecordIds) + filteredRecordIds = getIntersection(recordIds, filteredRecordIds) } } + if pagination != nil { + resultRecordIds, pageResp = paginate(filteredRecordIds, pagination) + } else { + resultRecordIds = filteredRecordIds + } + records := []registrytypes.Record{} for _, id := range resultRecordIds { record, err := k.GetRecordById(ctx, id) if err != nil { - return nil, err + return nil, nil, err } if record.Deleted { continue @@ -301,7 +328,7 @@ func (k Keeper) RecordsFromAttributes( records = append(records, record) } - return records, nil + return records, pageResp, nil } // TODO not recursive, and only should be if we want to support querying with whole sub-objects, @@ -717,6 +744,38 @@ func (k Keeper) tryTakeRecordRent(ctx sdk.Context, record registrytypes.Record) return k.SaveRecord(ctx, record) } +// paginate implements basic pagination over a list of objects +func paginate[T any](data []T, pagination *query.PageRequest) ([]T, *query.PageResponse) { + pageReq := initPageRequestDefaults(pagination) + + offset := pageReq.Offset + limit := pageReq.Limit + countTotal := pageReq.CountTotal + + totalItems := uint64(len(data)) + start := offset + end := offset + limit + + if start > totalItems { + if countTotal { + return []T{}, &query.PageResponse{Total: 0} + } else { + return []T{}, nil + } + } + if end > totalItems { + end = totalItems + } + + paginatedItems := data[start:end] + + if countTotal { + return paginatedItems, &query.PageResponse{Total: totalItems} + } else { + return paginatedItems, nil + } +} + func getIntersection(a []string, b []string) []string { result := []string{} if len(a) < len(b) { @@ -743,3 +802,26 @@ func contains(arr []string, str string) bool { } return false } + +// https://github.com/cosmos/cosmos-sdk/blob/v0.50.3/types/query/pagination.go#L141 +// initPageRequestDefaults initializes a PageRequest's defaults when those are not set. +func initPageRequestDefaults(pageRequest *query.PageRequest) *query.PageRequest { + // if the PageRequest is nil, use default PageRequest + if pageRequest == nil { + pageRequest = &query.PageRequest{} + } + + pageRequestCopy := *pageRequest + if len(pageRequestCopy.Key) == 0 { + pageRequestCopy.Key = nil + } + + if pageRequestCopy.Limit == 0 { + pageRequestCopy.Limit = query.DefaultLimit + + // count total results when the limit is zero/not supplied + pageRequestCopy.CountTotal = true + } + + return &pageRequestCopy +} diff --git a/x/registry/keeper/query_server.go b/x/registry/keeper/query_server.go index 17d2883d..b34e69ce 100644 --- a/x/registry/keeper/query_server.go +++ b/x/registry/keeper/query_server.go @@ -6,6 +6,7 @@ import ( errorsmod "cosmossdk.io/errors" sdk "github.com/cosmos/cosmos-sdk/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "github.com/cosmos/cosmos-sdk/types/query" registrytypes "git.vdb.to/cerc-io/laconicd/x/registry" ) @@ -39,20 +40,21 @@ func (qs queryServer) Records(c context.Context, req *registrytypes.QueryRecords all := req.GetAll() var records []registrytypes.Record + var pageResp *query.PageResponse var err error if len(attributes) > 0 { - records, err = qs.k.RecordsFromAttributes(ctx, attributes, all) + records, pageResp, err = qs.k.PaginatedRecordsFromAttributes(ctx, attributes, all, req.Pagination) if err != nil { return nil, err } } else { - records, err = qs.k.ListRecords(ctx) + records, pageResp, err = qs.k.PaginatedListRecords(ctx, req.Pagination) if err != nil { return nil, err } } - return ®istrytypes.QueryRecordsResponse{Records: records}, nil + return ®istrytypes.QueryRecordsResponse{Records: records, Pagination: pageResp}, nil } func (qs queryServer) GetRecord(c context.Context, req *registrytypes.QueryGetRecordRequest) (*registrytypes.QueryGetRecordResponse, error) {