cosmos-sdk/schema/indexer/start.go

package indexer

import (
	"context"
	"encoding/json"
	"fmt"
	"reflect"
	"sync"

	"cosmossdk.io/schema/addressutil"
	"cosmossdk.io/schema/appdata"
	"cosmossdk.io/schema/decoding"
	"cosmossdk.io/schema/logutil"
	"cosmossdk.io/schema/view"
)

// IndexingOptions are the options for starting the indexer manager.
type IndexingOptions struct {
	// Config is the user configuration for all indexing. It should generally be an instance map[string]interface{}
	// or json.RawMessage and match the json structure of IndexingConfig, or it can be an instance of IndexingConfig.
	// The manager will attempt to convert it to IndexingConfig.
	Config interface{}

	// Resolver is the decoder resolver that will be used to decode the data. It is required.
	Resolver decoding.DecoderResolver

	// SyncSource is a representation of the current state of key-value data to be used in a catch-up sync.
	// Catch-up syncs will be performed at initialization when necessary. SyncSource is optional but if
	// it is omitted, indexers will only be able to start indexing state from genesis.
	SyncSource decoding.SyncSource

	// Logger is the logger that indexers can use to write logs. It is optional.
	Logger logutil.Logger

	// Context is the context that indexers should use for shutdown signals via Context.Done(). It can also
	// be used to pass down other parameters to indexers if necessary. If it is omitted, context.Background
	// will be used.
	Context context.Context

	// AddressCodec is the address codec that indexers can use to encode and decode addresses. It should always be
	// provided, but if it is omitted, the indexer manager will use a default codec which encodes and decodes addresses
	// as hex strings.
	AddressCodec addressutil.AddressCodec

	// DoneWaitGroup is a wait group that all indexer manager go routines will wait on before returning when the context
	// is done.
	// It is optional.
	DoneWaitGroup *sync.WaitGroup
}

// IndexingConfig is the configuration of the indexer manager and contains the configuration for each indexer target.
type IndexingConfig struct {
	// Target is a map of named indexer targets to their configuration.
	Target map[string]Config `mapstructure:"target" toml:"target" json:"target" comment:"Target is a map of named indexer targets to their configuration."`

	// ChannelBufferSize is the buffer size of the channels used for buffering data sent to indexer go routines.
	// It defaults to 1024.
	ChannelBufferSize int `mapstructure:"channel_buffer_size" toml:"channel_buffer_size" json:"channel_buffer_size,omitempty" comment:"Buffer size of the channels used for buffering data sent to indexer go routines."`
}

// IndexingTarget returns the indexing target listener and associated data.
// The returned listener is the root listener to which app data should be sent.
type IndexingTarget struct {
	// Listener is the root listener to which app data should be sent.
	// It will do all processing in the background so updates should be sent synchronously.
	Listener appdata.Listener

	// ModuleFilter returns the root module filter which an app can use to exclude modules at the storage level,
	// if such a filter is set.
	ModuleFilter *ModuleFilterConfig

	IndexerInfos map[string]IndexerInfo
}

// IndexerInfo contains data returned by a specific indexer after initialization that maybe useful for the app.
type IndexerInfo struct {
	// View is the view returned by the indexer in its InitResult. It is optional and may be nil.
	View view.AppData
}

// StartIndexing starts the indexer manager with the given options. The state machine should write all relevant app data to
// the returned listener.
func StartIndexing(opts IndexingOptions) (IndexingTarget, error) {
	logger := opts.Logger
	if logger == nil {
		logger = logutil.NoopLogger{}
	}

	logger.Info("Starting indexing")

	cfg, err := unmarshalIndexingConfig(opts.Config)
	if err != nil {
		return IndexingTarget{}, err
	}

	ctx := opts.Context
	if ctx == nil {
		ctx = context.Background()
	}

	listeners := make([]appdata.Listener, 0, len(cfg.Target))
	indexerInfos := make(map[string]IndexerInfo, len(cfg.Target))

	for targetName, targetCfg := range cfg.Target {
		init, ok := indexerRegistry[targetCfg.Type]
		if !ok {
			return IndexingTarget{}, fmt.Errorf("indexer type %q not found", targetCfg.Type)
		}

		logger.Info("Starting indexer", "target_name", targetName, "type", targetCfg.Type)

		if targetCfg.Filter != nil {
			return IndexingTarget{}, fmt.Errorf("indexer filter options are not supported yet")
		}

		childLogger := logger
		if scopeableLogger, ok := logger.(logutil.ScopeableLogger); ok {
			childLogger = scopeableLogger.WithContext("indexer", targetName).(logutil.Logger)
		}

		targetCfg.Config, err = unmarshalIndexerCustomConfig(targetCfg.Config, init.ConfigType)
		if err != nil {
			return IndexingTarget{}, fmt.Errorf("failed to unmarshal indexer config for target %q: %v", targetName, err) //nolint:errorlint // we support go 1.12, so no error wrapping
		}

		initRes, err := init.InitFunc(InitParams{
			Config:       targetCfg,
			Context:      ctx,
			Logger:       childLogger,
			AddressCodec: opts.AddressCodec,
		})
		if err != nil {
			return IndexingTarget{}, err
		}

		listener := initRes.Listener
		listeners = append(listeners, listener)

		indexerInfos[targetName] = IndexerInfo{
			View: initRes.View,
		}
	}

	bufSize := 1024
	if cfg.ChannelBufferSize != 0 {
		bufSize = cfg.ChannelBufferSize
	}
	asyncOpts := appdata.AsyncListenerOptions{
		Context:       ctx,
		DoneWaitGroup: opts.DoneWaitGroup,
		BufferSize:    bufSize,
	}

	rootListener := appdata.AsyncListenerMux(
		asyncOpts,
		listeners...,
	)

	rootListener, err = decoding.Middleware(rootListener, opts.Resolver, decoding.MiddlewareOptions{})
	if err != nil {
		return IndexingTarget{}, err
	}
	rootListener = appdata.AsyncListener(asyncOpts, rootListener)

	return IndexingTarget{
		Listener:     rootListener,
		IndexerInfos: indexerInfos,
	}, nil
}

func unmarshalIndexingConfig(cfg interface{}) (*IndexingConfig, error) {
	if x, ok := cfg.(*IndexingConfig); ok {
		return x, nil
	}
	if x, ok := cfg.(IndexingConfig); ok {
		return &x, nil
	}

	var jsonBz []byte
	var err error

	switch cfg := cfg.(type) {
	case map[string]interface{}:
		jsonBz, err = json.Marshal(cfg)
		if err != nil {
			return nil, err
		}
	case json.RawMessage:
		jsonBz = cfg
	default:
		return nil, fmt.Errorf("can't convert %T to %T", cfg, IndexingConfig{})
	}

	var res IndexingConfig
	err = json.Unmarshal(jsonBz, &res)
	return &res, err
}

func unmarshalIndexerCustomConfig(cfg, expectedType interface{}) (interface{}, error) {
	typ := reflect.TypeOf(expectedType)
	if reflect.TypeOf(cfg).AssignableTo(typ) {
		return cfg, nil
	}

	res := reflect.New(typ).Interface()
	bz, err := json.Marshal(cfg)
	if err != nil {
		return nil, err
	}
	err = json.Unmarshal(bz, res)
	return reflect.ValueOf(res).Elem().Interface(), err
}