feat!: Store eth tx index separately (#1121)

* Store eth tx index separately Closes: #1075 Solution: - run a optional indexer service - adapt the json-rpc to the more efficient query changelog changelog fix lint fix backward compatibility fix lint timeout better strconv fix linter fix package name add cli command to index old tx fix for loop indexer cmd don't have access to local rpc workaround exceed block gas limit situation add unit tests for indexer refactor polish the indexer module Update server/config/toml.go Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com> improve comments share code between GetTxByEthHash and GetTxByIndex fix unit test Update server/indexer.go Co-authored-by: Freddy Caceres <facs95@gmail.com> * Apply suggestions from code review * test enable-indexer in integration test * fix go lint * address review suggestions * fix linter * address review suggestions - test indexer in backend unit test - add comments * fix build * fix test * service name Co-authored-by: Freddy Caceres <facs95@gmail.com> Co-authored-by: Federico Kunze Küllmer <31522760+fedekunze@users.noreply.github.com>
2022-08-11 22:49:05 +02:00
parent 737c1de694
commit 77ed4aa754
31 changed files with 1559 additions and 345 deletions
@@ -108,6 +108,8 @@ type JSONRPCConfig struct {
 	// MaxOpenConnections sets the maximum number of simultaneous connections
 	// for the server listener.
 	MaxOpenConnections int `mapstructure:"max-open-connections"`
+	// EnableIndexer defines if enable the custom indexer service.
+	EnableIndexer bool `mapstructure:"enable-indexer"`
 }

 // TLSConfig defines the certificate and matching private key for the server.
@@ -208,6 +210,7 @@ func DefaultJSONRPCConfig() *JSONRPCConfig {
 		HTTPIdleTimeout:     DefaultHTTPIdleTimeout,
 		AllowUnprotectedTxs: DefaultAllowUnprotectedTxs,
 		MaxOpenConnections:  DefaultMaxOpenConnections,
+		EnableIndexer:       false,
 	}
 }

@@ -312,6 +315,7 @@ func GetConfig(v *viper.Viper) Config {
 			HTTPTimeout:        v.GetDuration("json-rpc.http-timeout"),
 			HTTPIdleTimeout:    v.GetDuration("json-rpc.http-idle-timeout"),
 			MaxOpenConnections: v.GetInt("json-rpc.max-open-connections"),
+			EnableIndexer:      v.GetBool("json-rpc.enable-indexer"),
 		},
 		TLS: TLSConfig{
 			CertificatePath: v.GetString("tls.certificate-path"),
@@ -70,6 +70,9 @@ allow-unprotected-txs = {{ .JSONRPC.AllowUnprotectedTxs }}
 # for the server listener.
 max-open-connections = {{ .JSONRPC.MaxOpenConnections }}

+# EnableIndexer enables the custom transaction indexer for the EVM (ethereum transactions).
+enable-indexer = {{ .JSONRPC.EnableIndexer }}
+
 ###############################################################################
 ###                             TLS Configuration                           ###
 ###############################################################################
@@ -48,6 +48,7 @@ const (
 	JSONRPCHTTPIdleTimeout     = "json-rpc.http-idle-timeout"
 	JSONRPCAllowUnprotectedTxs = "json-rpc.allow-unprotected-txs"
 	JSONRPCMaxOpenConnections  = "json-rpc.max-open-connections"
+	JSONRPCEnableIndexer       = "json-rpc.enable-indexer"
 )

 // EVM flags
@@ -0,0 +1,112 @@
+package server
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/server"
+	"github.com/evmos/ethermint/indexer"
+	tmnode "github.com/tendermint/tendermint/node"
+	sm "github.com/tendermint/tendermint/state"
+	tmstore "github.com/tendermint/tendermint/store"
+)
+
+func NewIndexTxCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "index-eth-tx [forward|backward]",
+		Short: "Index historical eth txs",
+		Long: `Index historical eth txs, it only support two traverse direction to avoid creating gaps in the indexer db if using arbitrary block ranges:
+		- backward: index the blocks from the first indexed block to the earliest block in the chain.
+		- forward: index the blocks from the latest indexed block to latest block in the chain.
+		`,
+		Args: cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			serverCtx := server.GetServerContextFromCmd(cmd)
+			clientCtx, err := client.GetClientQueryContext(cmd)
+			if err != nil {
+				return err
+			}
+
+			direction := args[0]
+			if direction != "backward" && direction != "forward" {
+				return fmt.Errorf("unknown index direction, expect: backward|forward, got: %s", direction)
+			}
+
+			cfg := serverCtx.Config
+			home := cfg.RootDir
+			logger := serverCtx.Logger
+			idxDB, err := OpenIndexerDB(home, server.GetAppDBBackend(serverCtx.Viper))
+			if err != nil {
+				logger.Error("failed to open evm indexer DB", "error", err.Error())
+				return err
+			}
+			idxer := indexer.NewKVIndexer(idxDB, logger.With("module", "evmindex"), clientCtx)
+
+			// open local tendermint db, because the local rpc won't be available.
+			tmdb, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "blockstore", Config: cfg})
+			if err != nil {
+				return err
+			}
+			blockStore := tmstore.NewBlockStore(tmdb)
+
+			stateDB, err := tmnode.DefaultDBProvider(&tmnode.DBContext{ID: "state", Config: cfg})
+			if err != nil {
+				return err
+			}
+			stateStore := sm.NewStore(stateDB)
+
+			indexBlock := func(height int64) error {
+				blk := blockStore.LoadBlock(height)
+				if blk == nil {
+					return fmt.Errorf("block not found %d", height)
+				}
+				resBlk, err := stateStore.LoadABCIResponses(height)
+				if err != nil {
+					return err
+				}
+				if err := idxer.IndexBlock(blk, resBlk.DeliverTxs); err != nil {
+					return err
+				}
+				fmt.Println(height)
+				return nil
+			}
+
+			switch args[0] {
+			case "backward":
+				first, err := idxer.FirstIndexedBlock()
+				if err != nil {
+					return err
+				}
+				if first == -1 {
+					return fmt.Errorf("indexer db is empty")
+				}
+				for i := first - 1; i > 0; i-- {
+					if err := indexBlock(i); err != nil {
+						return err
+					}
+				}
+			case "forward":
+				latest, err := idxer.LastIndexedBlock()
+				if err != nil {
+					return err
+				}
+				if latest == -1 {
+					// start from genesis if empty
+					latest = 0
+				}
+				for i := latest + 1; i <= blockStore.Height(); i++ {
+					if err := indexBlock(i); err != nil {
+						return err
+					}
+				}
+			default:
+				return fmt.Errorf("unknown direction %s", args[0])
+			}
+
+			return nil
+		},
+	}
+	return cmd
+}
@@ -0,0 +1,109 @@
+package server
+
+import (
+	"context"
+	"time"
+
+	"github.com/tendermint/tendermint/libs/service"
+	rpcclient "github.com/tendermint/tendermint/rpc/client"
+	"github.com/tendermint/tendermint/types"
+
+	ethermint "github.com/evmos/ethermint/types"
+)
+
+const (
+	ServiceName = "EVMIndexerService"
+
+	NewBlockWaitTimeout = 60 * time.Second
+)
+
+// EVMIndexerService indexes transactions for json-rpc service.
+type EVMIndexerService struct {
+	service.BaseService
+
+	txIdxr ethermint.EVMTxIndexer
+	client rpcclient.Client
+}
+
+// NewEVMIndexerService returns a new service instance.
+func NewEVMIndexerService(
+	txIdxr ethermint.EVMTxIndexer,
+	client rpcclient.Client,
+) *EVMIndexerService {
+	is := &EVMIndexerService{txIdxr: txIdxr, client: client}
+	is.BaseService = *service.NewBaseService(nil, ServiceName, is)
+	return is
+}
+
+// OnStart implements service.Service by subscribing for new blocks
+// and indexing them by events.
+func (eis *EVMIndexerService) OnStart() error {
+	ctx := context.Background()
+	status, err := eis.client.Status(ctx)
+	if err != nil {
+		return err
+	}
+	latestBlock := status.SyncInfo.LatestBlockHeight
+	newBlockSignal := make(chan struct{}, 1)
+
+	// Use SubscribeUnbuffered here to ensure both subscriptions does not get
+	// canceled due to not pulling messages fast enough. Cause this might
+	// sometimes happen when there are no other subscribers.
+	blockHeadersChan, err := eis.client.Subscribe(
+		ctx,
+		ServiceName,
+		types.QueryForEvent(types.EventNewBlockHeader).String(),
+		0)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		for {
+			msg := <-blockHeadersChan
+			eventDataHeader := msg.Data.(types.EventDataNewBlockHeader)
+			if eventDataHeader.Header.Height > latestBlock {
+				latestBlock = eventDataHeader.Header.Height
+				// notify
+				select {
+				case newBlockSignal <- struct{}{}:
+				default:
+				}
+			}
+		}
+	}()
+
+	lastBlock, err := eis.txIdxr.LastIndexedBlock()
+	if err != nil {
+		return err
+	}
+	if lastBlock == -1 {
+		lastBlock = latestBlock
+	}
+	for {
+		if latestBlock <= lastBlock {
+			// nothing to index. wait for signal of new block
+			select {
+			case <-newBlockSignal:
+			case <-time.After(NewBlockWaitTimeout):
+			}
+			continue
+		}
+		for i := lastBlock + 1; i <= latestBlock; i++ {
+			block, err := eis.client.Block(ctx, &i)
+			if err != nil {
+				eis.Logger.Error("failed to fetch block", "height", i, "err", err)
+				break
+			}
+			blockResult, err := eis.client.BlockResults(ctx, &i)
+			if err != nil {
+				eis.Logger.Error("failed to fetch block result", "height", i, "err", err)
+				break
+			}
+			if err := eis.txIdxr.IndexBlock(block.Block, blockResult.TxsResults); err != nil {
+				eis.Logger.Error("failed to index block", "height", i, "err", err)
+			}
+			lastBlock = blockResult.Height
+		}
+	}
+}
@@ -15,10 +15,11 @@ import (
 	"github.com/evmos/ethermint/rpc"

 	"github.com/evmos/ethermint/server/config"
+	ethermint "github.com/evmos/ethermint/types"
 )

 // StartJSONRPC starts the JSON-RPC server
-func StartJSONRPC(ctx *server.Context, clientCtx client.Context, tmRPCAddr, tmEndpoint string, config *config.Config) (*http.Server, chan struct{}, error) {
+func StartJSONRPC(ctx *server.Context, clientCtx client.Context, tmRPCAddr, tmEndpoint string, config *config.Config, indexer ethermint.EVMTxIndexer) (*http.Server, chan struct{}, error) {
 	tmWsClient := ConnectTmWS(tmRPCAddr, tmEndpoint, ctx.Logger)

 	logger := ctx.Logger.With("module", "geth")
@@ -39,7 +40,7 @@ func StartJSONRPC(ctx *server.Context, clientCtx client.Context, tmRPCAddr, tmEn
 	allowUnprotectedTxs := config.JSONRPC.AllowUnprotectedTxs
 	rpcAPIArr := config.JSONRPC.API

-	apis := rpc.GetRPCAPIs(ctx, clientCtx, tmWsClient, allowUnprotectedTxs, rpcAPIArr)
+	apis := rpc.GetRPCAPIs(ctx, clientCtx, tmWsClient, allowUnprotectedTxs, indexer, rpcAPIArr)

 	for _, api := range apis {
 		if err := rpcServer.RegisterName(api.Namespace, api.Service); err != nil {
@@ -40,9 +40,11 @@ import (
 	servergrpc "github.com/cosmos/cosmos-sdk/server/grpc"
 	"github.com/cosmos/cosmos-sdk/server/types"

+	"github.com/evmos/ethermint/indexer"
 	ethdebug "github.com/evmos/ethermint/rpc/namespaces/ethereum/debug"
 	"github.com/evmos/ethermint/server/config"
 	srvflags "github.com/evmos/ethermint/server/flags"
+	ethermint "github.com/evmos/ethermint/types"
 )

 // StartCmd runs the service passed in, either stand-alone or in-process with
@@ -165,6 +167,7 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().Int32(srvflags.JSONRPCLogsCap, config.DefaultLogsCap, "Sets the max number of results can be returned from single `eth_getLogs` query")
 	cmd.Flags().Int32(srvflags.JSONRPCBlockRangeCap, config.DefaultBlockRangeCap, "Sets the max block range allowed for `eth_getLogs` query")
 	cmd.Flags().Int(srvflags.JSONRPCMaxOpenConnections, config.DefaultMaxOpenConnections, "Sets the maximum number of simultaneous connections for the server listener")
+	cmd.Flags().Bool(srvflags.JSONRPCEnableIndexer, false, "Enable the custom tx indexer for json-rpc")

 	cmd.Flags().String(srvflags.EVMTracer, config.DefaultEVMTracer, "the EVM tracer type to collect execution traces from the EVM transaction execution (json|struct|access_list|markdown)")
 	cmd.Flags().Uint64(srvflags.EVMMaxTxGasWanted, config.DefaultMaxTxGasWanted, "the gas wanted for each eth tx returned in ante handler in check tx mode")
@@ -323,13 +326,39 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty
 	// Add the tx service to the gRPC router. We only need to register this
 	// service if API or gRPC or JSONRPC is enabled, and avoid doing so in the general
 	// case, because it spawns a new local tendermint RPC client.
-	if config.API.Enable || config.GRPC.Enable || config.JSONRPC.Enable {
+	if config.API.Enable || config.GRPC.Enable || config.JSONRPC.Enable || config.JSONRPC.EnableIndexer {
 		clientCtx = clientCtx.WithClient(local.New(tmNode))

 		app.RegisterTxService(clientCtx)
 		app.RegisterTendermintService(clientCtx)
 	}

+	var idxer ethermint.EVMTxIndexer
+	if config.JSONRPC.EnableIndexer {
+		idxDB, err := OpenIndexerDB(home, server.GetAppDBBackend(ctx.Viper))
+		if err != nil {
+			logger.Error("failed to open evm indexer DB", "error", err.Error())
+			return err
+		}
+		idxLogger := ctx.Logger.With("module", "evmindex")
+		idxer = indexer.NewKVIndexer(idxDB, idxLogger, clientCtx)
+		indexerService := NewEVMIndexerService(idxer, clientCtx.Client)
+		indexerService.SetLogger(idxLogger)
+
+		errCh := make(chan error)
+		go func() {
+			if err := indexerService.Start(); err != nil {
+				errCh <- err
+			}
+		}()
+
+		select {
+		case err := <-errCh:
+			return err
+		case <-time.After(types.ServerStartTime): // assume server started successfully
+		}
+	}
+
 	var apiSrv *api.Server
 	if config.API.Enable {
 		genDoc, err := genDocProvider()
@@ -427,7 +456,7 @@ func startInProcess(ctx *server.Context, clientCtx client.Context, appCreator ty

 		tmEndpoint := "/websocket"
 		tmRPCAddr := cfg.RPC.ListenAddress
-		httpSrv, httpSrvDone, err = StartJSONRPC(ctx, clientCtx, tmRPCAddr, tmEndpoint, &config)
+		httpSrv, httpSrvDone, err = StartJSONRPC(ctx, clientCtx, tmRPCAddr, tmEndpoint, &config, idxer)
 		if err != nil {
 			return err
 		}
@@ -482,6 +511,12 @@ func openDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
 	return dbm.NewDB("application", backendType, dataDir)
 }

+// OpenIndexerDB opens the custom eth indexer db, using the same db backend as the main app
+func OpenIndexerDB(rootDir string, backendType dbm.BackendType) (dbm.DB, error) {
+	dataDir := filepath.Join(rootDir, "data")
+	return dbm.NewDB("evmindexer", backendType, dataDir)
+}
+
 func openTraceWriter(traceWriterFile string) (w io.Writer, err error) {
 	if traceWriterFile == "" {
 		return
@@ -45,6 +45,9 @@ func AddCommands(rootCmd *cobra.Command, defaultNodeHome string, appCreator type
 		sdkserver.ExportCmd(appExport, defaultNodeHome),
 		version.NewVersionCommand(),
 		sdkserver.NewRollbackCmd(defaultNodeHome),
+
+		// custom tx indexer command
+		NewIndexTxCmd(),
 	)
 }