feat(schema): indexing API (#20647)

schema/decoding/resolver.go

@@ -0,0 +1,66 @@
+package decoding
+
+import (
+	"sort"
+
+	"cosmossdk.io/schema"
+)
+
+// DecoderResolver is an interface that allows indexers to discover and use module decoders.
+type DecoderResolver interface {
+	// IterateAll iterates over all available module decoders.
+	IterateAll(func(moduleName string, cdc schema.ModuleCodec) error) error
+
+	// LookupDecoder looks up a specific module decoder.
+	LookupDecoder(moduleName string) (decoder schema.ModuleCodec, found bool, err error)
+}
+
+// ModuleSetDecoderResolver returns DecoderResolver that will discover modules implementing
+// DecodeableModule in the provided module set.
+func ModuleSetDecoderResolver(moduleSet map[string]interface{}) DecoderResolver {
+	return &moduleSetDecoderResolver{
+		moduleSet: moduleSet,
+	}
+}
+
+type moduleSetDecoderResolver struct {
+	moduleSet map[string]interface{}
+}
+
+func (a moduleSetDecoderResolver) IterateAll(f func(string, schema.ModuleCodec) error) error {
+	keys := make([]string, 0, len(a.moduleSet))
+	for k := range a.moduleSet {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		module := a.moduleSet[k]
+		dm, ok := module.(schema.HasModuleCodec)
+		if ok {
+			decoder, err := dm.ModuleCodec()
+			if err != nil {
+				return err
+			}
+			err = f(k, decoder)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (a moduleSetDecoderResolver) LookupDecoder(moduleName string) (schema.ModuleCodec, bool, error) {
+	mod, ok := a.moduleSet[moduleName]
+	if !ok {
+		return schema.ModuleCodec{}, false, nil
+	}
+
+	dm, ok := mod.(schema.HasModuleCodec)
+	if !ok {
+		return schema.ModuleCodec{}, false, nil
+	}
+
+	decoder, err := dm.ModuleCodec()
+	return decoder, true, err
+}

schema/decoding/resolver_test.go

@@ -0,0 +1,124 @@
+package decoding
+
+import (
+	"fmt"
+	"testing"
+
+	"cosmossdk.io/schema"
+)
+
+type modA struct{}
+
+func (m modA) ModuleCodec() (schema.ModuleCodec, error) {
+	return schema.ModuleCodec{
+		Schema: schema.ModuleSchema{ObjectTypes: []schema.ObjectType{{Name: "A"}}},
+	}, nil
+}
+
+type modB struct{}
+
+func (m modB) ModuleCodec() (schema.ModuleCodec, error) {
+	return schema.ModuleCodec{
+		Schema: schema.ModuleSchema{ObjectTypes: []schema.ObjectType{{Name: "B"}}},
+	}, nil
+}
+
+type modC struct{}
+
+var moduleSet = map[string]interface{}{
+	"modA": modA{},
+	"modB": modB{},
+	"modC": modC{},
+}
+
+var resolver = ModuleSetDecoderResolver(moduleSet)
+
+func TestModuleSetDecoderResolver_IterateAll(t *testing.T) {
+	objectTypes := map[string]bool{}
+	err := resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error {
+		objectTypes[cdc.Schema.ObjectTypes[0].Name] = true
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if len(objectTypes) != 2 {
+		t.Fatalf("expected 2 object types, got %d", len(objectTypes))
+	}
+
+	if !objectTypes["A"] {
+		t.Fatalf("expected object type A")
+	}
+
+	if !objectTypes["B"] {
+		t.Fatalf("expected object type B")
+	}
+}
+
+func TestModuleSetDecoderResolver_LookupDecoder(t *testing.T) {
+	decoder, found, err := resolver.LookupDecoder("modA")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !found {
+		t.Fatalf("expected to find decoder for modA")
+	}
+
+	if decoder.Schema.ObjectTypes[0].Name != "A" {
+		t.Fatalf("expected object type A, got %s", decoder.Schema.ObjectTypes[0].Name)
+	}
+
+	decoder, found, err = resolver.LookupDecoder("modB")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !found {
+		t.Fatalf("expected to find decoder for modB")
+	}
+
+	if decoder.Schema.ObjectTypes[0].Name != "B" {
+		t.Fatalf("expected object type B, got %s", decoder.Schema.ObjectTypes[0].Name)
+	}
+
+	decoder, found, err = resolver.LookupDecoder("modC")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if found {
+		t.Fatalf("expected not to find decoder")
+	}
+
+	decoder, found, err = resolver.LookupDecoder("modD")
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if found {
+		t.Fatalf("expected not to find decoder")
+	}
+}
+
+type modD struct{}
+
+func (m modD) ModuleCodec() (schema.ModuleCodec, error) {
+	return schema.ModuleCodec{}, fmt.Errorf("an error")
+}
+
+func TestModuleSetDecoderResolver_IterateAll_Error(t *testing.T) {
+	resolver := ModuleSetDecoderResolver(map[string]interface{}{
+		"modD": modD{},
+	})
+	err := resolver.IterateAll(func(moduleName string, cdc schema.ModuleCodec) error {
+		if moduleName == "modD" {
+			t.Fatalf("expected error")
+		}
+		return nil
+	})
+	if err == nil {
+		t.Fatalf("expected error")
+	}
+}

schema/decoding/sync.go

@@ -0,0 +1,8 @@
+package decoding
+
+// SyncSource is an interface that allows indexers to start indexing modules with pre-existing state.
+// It should generally be a wrapper around the key-value store.
+type SyncSource interface {
+	// IterateAllKVPairs iterates over all key-value pairs for a given module.
+	IterateAllKVPairs(moduleName string, fn func(key, value []byte) error) error
+}

schema/indexer/README.md

@@ -0,0 +1,15 @@
+# Indexer Framework
+
+# Defining an Indexer
+
+Indexer implementations should be registered with the `indexer.Register` function with a unique type name. Indexers take the configuration options defined by `indexer.Config` which defines a common set of configuration options as well as indexer-specific options under the `config` sub-key. Indexers do not need to manage the common filtering options specified in `Config` - the indexer manager will manage these for the indexer. Indexer implementations just need to return a correct `InitResult` response.
+
+# Integrating the Indexer Manager
+
+The indexer manager should be used for managing all indexers and should be integrated directly with applications wishing to support indexing. The `StartManager` function is used to start the manager. The configuration options for the manager and all indexer targets should be passed as the ManagerOptions.Config field and should match the json structure of ManagerConfig. An example configuration section in `app.toml` might look like this:
+
+```toml
+[indexer.target.postgres]
+type = "postgres"
+config.database_url = "postgres://user:password@localhost:5432/dbname"
+```

schema/indexer/indexer.go

@@ -0,0 +1,78 @@
+package indexer
+
+import (
+	"context"
+
+	"cosmossdk.io/schema/appdata"
+	"cosmossdk.io/schema/logutil"
+)
+
+// Config species the configuration passed to an indexer initialization function.
+// It includes both common configuration options related to include or excluding
+// parts of the data stream as well as indexer specific options under the config
+// subsection.
+//
+// NOTE: it is an error for an indexer to change its common options, such as adding
+// or removing indexed modules, after the indexer has been initialized because this
+// could result in an inconsistent state.
+type Config struct {
+	// Type is the name of the indexer type as registered with Register.
+	Type string `json:"type"`
+
+	// Config are the indexer specific config options specified by the user.
+	Config map[string]interface{} `json:"config"`
+
+	// ExcludeState specifies that the indexer will not receive state updates.
+	ExcludeState bool `json:"exclude_state"`
+
+	// ExcludeEvents specifies that the indexer will not receive events.
+	ExcludeEvents bool `json:"exclude_events"`
+
+	// ExcludeTxs specifies that the indexer will not receive transaction's.
+	ExcludeTxs bool `json:"exclude_txs"`
+
+	// ExcludeBlockHeaders specifies that the indexer will not receive block headers,
+	// although it will still receive StartBlock and Commit callbacks, just without
+	// the header data.
+	ExcludeBlockHeaders bool `json:"exclude_block_headers"`
+
+	// IncludeModules specifies a list of modules whose state the indexer will
+	// receive state updates for.
+	// Only one of include or exclude modules should be specified.
+	IncludeModules []string `json:"include_modules"`
+
+	// ExcludeModules specifies a list of modules whose state the indexer will not
+	// receive state updates for.
+	// Only one of include or exclude modules should be specified.
+	ExcludeModules []string `json:"exclude_modules"`
+}
+
+type InitFunc = func(InitParams) (InitResult, error)
+
+// InitParams is the input to the indexer initialization function.
+type InitParams struct {
+	// Config is the indexer config.
+	Config Config
+
+	// Context is the context that the indexer should use to listen for a shutdown signal via Context.Done(). Other
+	// parameters may also be passed through context from the app if necessary.
+	Context context.Context
+
+	// Logger is a logger the indexer can use to write log messages.
+	Logger logutil.Logger
+}
+
+// InitResult is the indexer initialization result and includes the indexer's listener implementation.
+type InitResult struct {
+	// Listener is the indexer's app data listener.
+	Listener appdata.Listener
+
+	// LastBlockPersisted indicates the last block that the indexer persisted (if it is persisting data). It
+	// should be 0 if the indexer has no data stored and wants to start syncing state. It should be -1 if the indexer
+	// does not care to persist state at all and is just listening for some other streaming purpose. If the indexer
+	// has persisted state and has missed some blocks, a runtime error will occur to prevent the indexer from continuing
+	// in an invalid state. If an indexer starts indexing after a chain's genesis (returning 0), the indexer manager
+	// will attempt to perform a catch-up sync of state. Historical events will not be replayed, but an accurate
+	// representation of the current state at the height at which indexing began can be reproduced.
+	LastBlockPersisted int64
+}

schema/indexer/manager.go

@@ -0,0 +1,44 @@
+package indexer
+
+import (
+	"context"
+
+	"cosmossdk.io/schema/appdata"
+	"cosmossdk.io/schema/decoding"
+	"cosmossdk.io/schema/logutil"
+)
+
+// ManagerOptions are the options for starting the indexer manager.
+type ManagerOptions struct {
+	// Config is the user configuration for all indexing. It should generally be an instance of map[string]interface{}
+	// and match the json structure of ManagerConfig. The manager will attempt to convert it to ManagerConfig.
+	Config interface{}
+
+	// Resolver is the decoder resolver that will be used to decode the data. It is required.
+	Resolver decoding.DecoderResolver
+
+	// SyncSource is a representation of the current state of key-value data to be used in a catch-up sync.
+	// Catch-up syncs will be performed at initialization when necessary. SyncSource is optional but if
+	// it is omitted, indexers will only be able to start indexing state from genesis.
+	SyncSource decoding.SyncSource
+
+	// Logger is the logger that indexers can use to write logs. It is optional.
+	Logger logutil.Logger
+
+	// Context is the context that indexers should use for shutdown signals via Context.Done(). It can also
+	// be used to pass down other parameters to indexers if necessary. If it is omitted, context.Background
+	// will be used.
+	Context context.Context
+}
+
+// ManagerConfig is the configuration of the indexer manager and contains the configuration for each indexer target.
+type ManagerConfig struct {
+	// Target is a map of named indexer targets to their configuration.
+	Target map[string]Config
+}
+
+// StartManager starts the indexer manager with the given options. The state machine should write all relevant app data to
+// the returned listener.
+func StartManager(opts ManagerOptions) (appdata.Listener, error) {
+	panic("TODO: this will be implemented in a follow-up PR, this function is just a stub to demonstrate the API")
+}

schema/indexer/registry.go

@@ -0,0 +1,14 @@
+package indexer
+
+import "fmt"
+
+// Register registers an indexer type with the given initialization function.
+func Register(indexerType string, initFunc InitFunc) {
+	if _, ok := indexerRegistry[indexerType]; ok {
+		panic(fmt.Sprintf("indexer %s already registered", indexerType))
+	}
+
+	indexerRegistry[indexerType] = initFunc
+}
+
+var indexerRegistry = map[string]InitFunc{}

schema/indexer/registry_test.go

@@ -0,0 +1,26 @@
+package indexer
+
+import "testing"
+
+func TestRegister(t *testing.T) {
+	Register("test", func(params InitParams) (InitResult, error) {
+		return InitResult{}, nil
+	})
+
+	if indexerRegistry["test"] == nil {
+		t.Fatalf("expected to find indexer")
+	}
+
+	if indexerRegistry["test2"] != nil {
+		t.Fatalf("expected not to find indexer")
+	}
+
+	defer func() {
+		if r := recover(); r == nil {
+			t.Fatalf("expected to panic")
+		}
+	}()
+	Register("test", func(params InitParams) (InitResult, error) {
+		return InitResult{}, nil
+	})
+}

schema/logutil/logger.go

@@ -0,0 +1,35 @@
+// Package logutil defines the Logger interface expected by indexer implementations.
+// It is implemented by cosmossdk.io/log which is not imported to minimize dependencies.
+package logutil
+
+// Logger is the logger interface expected by indexer implementations.
+type Logger interface {
+	// Info takes a message and a set of key/value pairs and logs with level INFO.
+	// The key of the tuple must be a string.
+	Info(msg string, keyVals ...interface{})
+
+	// Warn takes a message and a set of key/value pairs and logs with level WARN.
+	// The key of the tuple must be a string.
+	Warn(msg string, keyVals ...interface{})
+
+	// Error takes a message and a set of key/value pairs and logs with level ERR.
+	// The key of the tuple must be a string.
+	Error(msg string, keyVals ...interface{})
+
+	// Debug takes a message and a set of key/value pairs and logs with level DEBUG.
+	// The key of the tuple must be a string.
+	Debug(msg string, keyVals ...interface{})
+}
+
+// NoopLogger is a logger that doesn't do anything.
+type NoopLogger struct{}
+
+func (n NoopLogger) Info(string, ...interface{}) {}
+
+func (n NoopLogger) Warn(string, ...interface{}) {}
+
+func (n NoopLogger) Error(string, ...interface{}) {}
+
+func (n NoopLogger) Debug(string, ...interface{}) {}
+
+var _ Logger = NoopLogger{}